fsmonitor: refresh pywatchman with upstream

This commit vendors pywatchman commit 259dc66dc9591f9b7ce76d0275bb1065f390c9b1
from upstream without modifications. The previously vendored pywatchman
from changeset 16f4b341288d was from Git commit c77452.

This commit effectively undoes the following Mercurial changesets:

* dd35abc409ee fsmonitor: correct an error message
* b1f62cd39b5c fsmonitor: layer on another hack in bser.c for os.stat()
  compat (issue5811)
* c31ce080eb75 py3: convert arguments, cwd and env to native strings when
  spawning subprocess
* 876494fd967d cleanup: delete lots of unused local variables
* 57264906a996 watchman: add the possibility to set the exact watchman
  binary location

The newly-vendored code has support for specifying the binary location,
so 57264906a996 does not need applied. But we do need to modify our
code to specify a proper argument name.

876494fd967d is not important, so it will be ignored.

c31ce080eb75 globally changed the code base to always pass
str to subprocess. But pywatchman's code is Python 3 clean, so
we don't need to do this.

This leaves dd35abc409ee and b1f62cd39b5c, which will be re-applied in
subsequent commits.

Differential Revision: https://phab.mercurial-scm.org/D7201

hgext/fsmonitor/pywatchman/__init__.py

@@ -1,1035 +1,1195 b''
 # Copyright 2014-present Facebook, Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 #  * Redistributions of source code must retain the above copyright notice,
 #    this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright notice,
 #    this list of conditions and the following disclaimer in the documentation
 #    and/or other materials provided with the distribution.
 #
 #  * Neither the name Facebook nor the names of its contributors may be used to
 #    endorse or promote products derived from this software without specific
 #    prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
 
 import inspect
 import math
 import os
 import socket
 import subprocess
 import time
 
+from . import capabilities, compat, encoding, load
+
+
 # Sometimes it's really hard to get Python extensions to compile,
 # so fall back to a pure Python implementation.
 try:
     from . import bser
+
     # Demandimport causes modules to be loaded lazily. Force the load now
     # so that we can fall back on pybser if bser doesn't exist
     bser.pdu_info
 except ImportError:
     from . import pybser as bser
 
-from mercurial.utils import (
-    procutil,
-)
 
-from mercurial import (
-    pycompat,
-)
-
-from . import (
-    capabilities,
-    compat,
-    encoding,
-    load,
-)
-
-
-if os.name == 'nt':
+if os.name == "nt":
     import ctypes
     import ctypes.wintypes
 
     wintypes = ctypes.wintypes
     GENERIC_READ = 0x80000000
     GENERIC_WRITE = 0x40000000
     FILE_FLAG_OVERLAPPED = 0x40000000
     OPEN_EXISTING = 3
-    INVALID_HANDLE_VALUE = -1
+    INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value
     FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000
     FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100
     FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200
     WAIT_FAILED = 0xFFFFFFFF
     WAIT_TIMEOUT = 0x00000102
     WAIT_OBJECT_0 = 0x00000000
     WAIT_IO_COMPLETION = 0x000000C0
     INFINITE = 0xFFFFFFFF
 
     # Overlapped I/O operation is in progress. (997)
     ERROR_IO_PENDING = 0x000003E5
 
     # The pointer size follows the architecture
     # We use WPARAM since this type is already conditionally defined
     ULONG_PTR = ctypes.wintypes.WPARAM
 
     class OVERLAPPED(ctypes.Structure):
         _fields_ = [
-            ("Internal", ULONG_PTR), ("InternalHigh", ULONG_PTR),
-            ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD),
-            ("hEvent", wintypes.HANDLE)
+            ("Internal", ULONG_PTR),
+            ("InternalHigh", ULONG_PTR),
+            ("Offset", wintypes.DWORD),
+            ("OffsetHigh", wintypes.DWORD),
+            ("hEvent", wintypes.HANDLE),
         ]
 
         def __init__(self):
             self.Internal = 0
             self.InternalHigh = 0
             self.Offset = 0
             self.OffsetHigh = 0
             self.hEvent = 0
 
     LPDWORD = ctypes.POINTER(wintypes.DWORD)
 
     CreateFile = ctypes.windll.kernel32.CreateFileA
-    CreateFile.argtypes = [wintypes.LPSTR, wintypes.DWORD, wintypes.DWORD,
-                           wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD,
-                           wintypes.HANDLE]
+    CreateFile.argtypes = [
+        wintypes.LPSTR,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        wintypes.HANDLE,
+    ]
     CreateFile.restype = wintypes.HANDLE
 
     CloseHandle = ctypes.windll.kernel32.CloseHandle
     CloseHandle.argtypes = [wintypes.HANDLE]
     CloseHandle.restype = wintypes.BOOL
 
     ReadFile = ctypes.windll.kernel32.ReadFile
-    ReadFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
-                         LPDWORD, ctypes.POINTER(OVERLAPPED)]
+    ReadFile.argtypes = [
+        wintypes.HANDLE,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        LPDWORD,
+        ctypes.POINTER(OVERLAPPED),
+    ]
     ReadFile.restype = wintypes.BOOL
 
     WriteFile = ctypes.windll.kernel32.WriteFile
-    WriteFile.argtypes = [wintypes.HANDLE, wintypes.LPVOID, wintypes.DWORD,
-                          LPDWORD, ctypes.POINTER(OVERLAPPED)]
+    WriteFile.argtypes = [
+        wintypes.HANDLE,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        LPDWORD,
+        ctypes.POINTER(OVERLAPPED),
+    ]
     WriteFile.restype = wintypes.BOOL
 
     GetLastError = ctypes.windll.kernel32.GetLastError
     GetLastError.argtypes = []
     GetLastError.restype = wintypes.DWORD
 
     SetLastError = ctypes.windll.kernel32.SetLastError
     SetLastError.argtypes = [wintypes.DWORD]
     SetLastError.restype = None
 
     FormatMessage = ctypes.windll.kernel32.FormatMessageA
-    FormatMessage.argtypes = [wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD,
-                              wintypes.DWORD, ctypes.POINTER(wintypes.LPSTR),
-                              wintypes.DWORD, wintypes.LPVOID]
+    FormatMessage.argtypes = [
+        wintypes.DWORD,
+        wintypes.LPVOID,
+        wintypes.DWORD,
+        wintypes.DWORD,
+        ctypes.POINTER(wintypes.LPSTR),
+        wintypes.DWORD,
+        wintypes.LPVOID,
+    ]
     FormatMessage.restype = wintypes.DWORD
 
     LocalFree = ctypes.windll.kernel32.LocalFree
 
     GetOverlappedResult = ctypes.windll.kernel32.GetOverlappedResult
-    GetOverlappedResult.argtypes = [wintypes.HANDLE,
-                                    ctypes.POINTER(OVERLAPPED), LPDWORD,
-                                    wintypes.BOOL]
+    GetOverlappedResult.argtypes = [
+        wintypes.HANDLE,
+        ctypes.POINTER(OVERLAPPED),
+        LPDWORD,
+        wintypes.BOOL,
+    ]
     GetOverlappedResult.restype = wintypes.BOOL
 
-    GetOverlappedResultEx = getattr(ctypes.windll.kernel32,
-                                    'GetOverlappedResultEx', None)
+    GetOverlappedResultEx = getattr(
+        ctypes.windll.kernel32, "GetOverlappedResultEx", None
+    )
     if GetOverlappedResultEx is not None:
-        GetOverlappedResultEx.argtypes = [wintypes.HANDLE,
-                                          ctypes.POINTER(OVERLAPPED), LPDWORD,
-                                          wintypes.DWORD, wintypes.BOOL]
+        GetOverlappedResultEx.argtypes = [
+            wintypes.HANDLE,
+            ctypes.POINTER(OVERLAPPED),
+            LPDWORD,
+            wintypes.DWORD,
+            wintypes.BOOL,
+        ]
         GetOverlappedResultEx.restype = wintypes.BOOL
 
     WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx
-    WaitForSingleObjectEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL]
+    WaitForSingleObjectEx.argtypes = [
+        wintypes.HANDLE,
+        wintypes.DWORD,
+        wintypes.BOOL,
+    ]
     WaitForSingleObjectEx.restype = wintypes.DWORD
 
     CreateEvent = ctypes.windll.kernel32.CreateEventA
-    CreateEvent.argtypes = [LPDWORD, wintypes.BOOL, wintypes.BOOL,
-                            wintypes.LPSTR]
+    CreateEvent.argtypes = [
+        LPDWORD,
+        wintypes.BOOL,
+        wintypes.BOOL,
+        wintypes.LPSTR,
+    ]
     CreateEvent.restype = wintypes.HANDLE
 
     # Windows Vista is the minimum supported client for CancelIoEx.
     CancelIoEx = ctypes.windll.kernel32.CancelIoEx
     CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)]
     CancelIoEx.restype = wintypes.BOOL
 
 # 2 bytes marker, 1 byte int size, 8 bytes int64 value
 sniff_len = 13
 
 # This is a helper for debugging the client.
 _debugging = False
 if _debugging:
 
     def log(fmt, *args):
-        print('[%s] %s' %
-              (time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
-               fmt % args[:]))
+        print(
+            "[%s] %s"
+            % (
+                time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()),
+                fmt % args[:],
+            )
+        )
+
+
 else:
 
     def log(fmt, *args):
         pass
 
 
 def _win32_strerror(err):
     """ expand a win32 error code into a human readable message """
 
     # FormatMessage will allocate memory and assign it here
     buf = ctypes.c_char_p()
     FormatMessage(
-        FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER
-        | FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None)
+        FORMAT_MESSAGE_FROM_SYSTEM
+        | FORMAT_MESSAGE_ALLOCATE_BUFFER
+        | FORMAT_MESSAGE_IGNORE_INSERTS,
+        None,
+        err,
+        0,
+        buf,
+        0,
+        None,
+    )
     try:
         return buf.value
     finally:
         LocalFree(buf)
 
 
 class WatchmanError(Exception):
     def __init__(self, msg=None, cmd=None):
         self.msg = msg
         self.cmd = cmd
 
     def setCommand(self, cmd):
         self.cmd = cmd
 
     def __str__(self):
         if self.cmd:
-            return '%s, while executing %s' % (self.msg, self.cmd)
+            return "%s, while executing %s" % (self.msg, self.cmd)
         return self.msg
 
 
+class BSERv1Unsupported(WatchmanError):
+    pass
+
+
+class UseAfterFork(WatchmanError):
+    pass
+
+
 class WatchmanEnvironmentError(WatchmanError):
     def __init__(self, msg, errno, errmsg, cmd=None):
         super(WatchmanEnvironmentError, self).__init__(
-            '{0}: errno={1} errmsg={2}'.format(msg, errno, errmsg),
-            cmd)
+            "{0}: errno={1} errmsg={2}".format(msg, errno, errmsg), cmd
+        )
 
 
 class SocketConnectError(WatchmanError):
     def __init__(self, sockpath, exc):
         super(SocketConnectError, self).__init__(
-            'unable to connect to %s: %s' % (sockpath, exc))
+            "unable to connect to %s: %s" % (sockpath, exc)
+        )
         self.sockpath = sockpath
         self.exc = exc
 
 
 class SocketTimeout(WatchmanError):
     """A specialized exception raised for socket timeouts during communication to/from watchman.
        This makes it easier to implement non-blocking loops as callers can easily distinguish
        between a routine timeout and an actual error condition.
 
        Note that catching WatchmanError will also catch this as it is a super-class, so backwards
        compatibility in exception handling is preserved.
     """
 
 
 class CommandError(WatchmanError):
     """error returned by watchman
 
     self.msg is the message returned by watchman.
     """
+
     def __init__(self, msg, cmd=None):
         super(CommandError, self).__init__(
-            'watchman command error: %s' % (msg, ),
-            cmd,
+            "watchman command error: %s" % (msg,), cmd
         )
 
 
 class Transport(object):
     """ communication transport to the watchman server """
+
     buf = None
 
     def close(self):
         """ tear it down """
         raise NotImplementedError()
 
     def readBytes(self, size):
         """ read size bytes """
         raise NotImplementedError()
 
     def write(self, buf):
         """ write some data """
         raise NotImplementedError()
 
     def setTimeout(self, value):
         pass
 
     def readLine(self):
         """ read a line
         Maintains its own buffer, callers of the transport should not mix
         calls to readBytes and readLine.
         """
         if self.buf is None:
             self.buf = []
 
         # Buffer may already have a line if we've received unilateral
         # response(s) from the server
         if len(self.buf) == 1 and b"\n" in self.buf[0]:
             (line, b) = self.buf[0].split(b"\n", 1)
             self.buf = [b]
             return line
 
         while True:
             b = self.readBytes(4096)
             if b"\n" in b:
-                result = b''.join(self.buf)
+                result = b"".join(self.buf)
                 (line, b) = b.split(b"\n", 1)
                 self.buf = [b]
                 return result + line
             self.buf.append(b)
 
 
 class Codec(object):
     """ communication encoding for the watchman server """
+
     transport = None
 
     def __init__(self, transport):
         self.transport = transport
 
     def receive(self):
         raise NotImplementedError()
 
     def send(self, *args):
         raise NotImplementedError()
 
     def setTimeout(self, value):
         self.transport.setTimeout(value)
 
 
 class UnixSocketTransport(Transport):
     """ local unix domain socket transport """
+
     sock = None
 
-    def __init__(self, sockpath, timeout, watchman_exe):
+    def __init__(self, sockpath, timeout):
         self.sockpath = sockpath
         self.timeout = timeout
 
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
         try:
             sock.settimeout(self.timeout)
             sock.connect(self.sockpath)
             self.sock = sock
         except socket.error as e:
             sock.close()
             raise SocketConnectError(self.sockpath, e)
 
     def close(self):
-        self.sock.close()
-        self.sock = None
+        if self.sock:
+            self.sock.close()
+            self.sock = None
 
     def setTimeout(self, value):
         self.timeout = value
         self.sock.settimeout(self.timeout)
 
     def readBytes(self, size):
         try:
             buf = [self.sock.recv(size)]
             if not buf[0]:
-                raise WatchmanError('empty watchman response')
+                raise WatchmanError("empty watchman response")
             return buf[0]
         except socket.timeout:
-            raise SocketTimeout('timed out waiting for response')
+            raise SocketTimeout("timed out waiting for response")
 
     def write(self, data):
         try:
             self.sock.sendall(data)
         except socket.timeout:
-            raise SocketTimeout('timed out sending query command')
+            raise SocketTimeout("timed out sending query command")
 
 
 def _get_overlapped_result_ex_impl(pipe, olap, nbytes, millis, alertable):
     """ Windows 7 and earlier does not support GetOverlappedResultEx. The
     alternative is to use GetOverlappedResult and wait for read or write
     operation to complete. This is done be using CreateEvent and
     WaitForSingleObjectEx. CreateEvent, WaitForSingleObjectEx
     and GetOverlappedResult are all part of Windows API since WindowsXP.
     This is the exact same implementation that can be found in the watchman
     source code (see get_overlapped_result_ex_impl in stream_win.c). This
     way, maintenance should be simplified.
     """
-    log('Preparing to wait for maximum %dms', millis )
+    log("Preparing to wait for maximum %dms", millis)
     if millis != 0:
         waitReturnCode = WaitForSingleObjectEx(olap.hEvent, millis, alertable)
         if waitReturnCode == WAIT_OBJECT_0:
             # Event is signaled, overlapped IO operation result should be available.
             pass
         elif waitReturnCode == WAIT_IO_COMPLETION:
             # WaitForSingleObjectEx returnes because the system added an I/O completion
             # routine or an asynchronous procedure call (APC) to the thread queue.
             SetLastError(WAIT_IO_COMPLETION)
             pass
         elif waitReturnCode == WAIT_TIMEOUT:
             # We reached the maximum allowed wait time, the IO operation failed
             # to complete in timely fashion.
             SetLastError(WAIT_TIMEOUT)
             return False
         elif waitReturnCode == WAIT_FAILED:
             # something went wrong calling WaitForSingleObjectEx
             err = GetLastError()
-            log('WaitForSingleObjectEx failed: %s', _win32_strerror(err))
+            log("WaitForSingleObjectEx failed: %s", _win32_strerror(err))
             return False
         else:
             # unexpected situation deserving investigation.
             err = GetLastError()
-            log('Unexpected error: %s', _win32_strerror(err))
+            log("Unexpected error: %s", _win32_strerror(err))
             return False
 
     return GetOverlappedResult(pipe, olap, nbytes, False)
 
 
 class WindowsNamedPipeTransport(Transport):
     """ connect to a named pipe """
 
-    def __init__(self, sockpath, timeout, watchman_exe):
+    def __init__(self, sockpath, timeout):
         self.sockpath = sockpath
         self.timeout = int(math.ceil(timeout * 1000))
         self._iobuf = None
 
-        self.pipe = CreateFile(sockpath, GENERIC_READ | GENERIC_WRITE, 0, None,
-                               OPEN_EXISTING, FILE_FLAG_OVERLAPPED, None)
+        if compat.PYTHON3:
+            sockpath = os.fsencode(sockpath)
+        self.pipe = CreateFile(
+            sockpath,
+            GENERIC_READ | GENERIC_WRITE,
+            0,
+            None,
+            OPEN_EXISTING,
+            FILE_FLAG_OVERLAPPED,
+            None,
+        )
 
-        if self.pipe == INVALID_HANDLE_VALUE:
+        err = GetLastError()
+        if self.pipe == INVALID_HANDLE_VALUE or self.pipe == 0:
             self.pipe = None
-            self._raise_win_err('failed to open pipe %s' % sockpath,
-                                GetLastError())
+            raise SocketConnectError(self.sockpath, self._make_win_err("", err))
 
         # event for the overlapped I/O operations
         self._waitable = CreateEvent(None, True, False, None)
+        err = GetLastError()
         if self._waitable is None:
-            self._raise_win_err('CreateEvent failed', GetLastError())
+            self._raise_win_err("CreateEvent failed", err)
 
         self._get_overlapped_result_ex = GetOverlappedResultEx
-        if (os.getenv('WATCHMAN_WIN7_COMPAT') == '1' or
-            self._get_overlapped_result_ex is None):
+        if (
+            os.getenv("WATCHMAN_WIN7_COMPAT") == "1"
+            or self._get_overlapped_result_ex is None
+        ):
             self._get_overlapped_result_ex = _get_overlapped_result_ex_impl
 
     def _raise_win_err(self, msg, err):
-        raise IOError('%s win32 error code: %d %s' %
-                      (msg, err, _win32_strerror(err)))
+        raise self._make_win_err(msg, err)
+
+    def _make_win_err(self, msg, err):
+        return IOError(
+            "%s win32 error code: %d %s" % (msg, err, _win32_strerror(err))
+        )
 
     def close(self):
         if self.pipe:
-            log('Closing pipe')
+            log("Closing pipe")
             CloseHandle(self.pipe)
         self.pipe = None
 
         if self._waitable is not None:
             # We release the handle for the event
             CloseHandle(self._waitable)
         self._waitable = None
 
     def setTimeout(self, value):
         # convert to milliseconds
         self.timeout = int(value * 1000)
 
     def readBytes(self, size):
         """ A read can block for an unbounded amount of time, even if the
             kernel reports that the pipe handle is signalled, so we need to
             always perform our reads asynchronously
         """
 
         # try to satisfy the read from any buffered data
         if self._iobuf:
             if size >= len(self._iobuf):
                 res = self._iobuf
                 self.buf = None
                 return res
             res = self._iobuf[:size]
             self._iobuf = self._iobuf[size:]
             return res
 
         # We need to initiate a read
         buf = ctypes.create_string_buffer(size)
         olap = OVERLAPPED()
         olap.hEvent = self._waitable
 
-        log('made read buff of size %d', size)
+        log("made read buff of size %d", size)
 
         # ReadFile docs warn against sending in the nread parameter for async
         # operations, so we always collect it via GetOverlappedResultEx
         immediate = ReadFile(self.pipe, buf, size, None, olap)
 
         if not immediate:
             err = GetLastError()
             if err != ERROR_IO_PENDING:
-                self._raise_win_err('failed to read %d bytes' % size,
-                                    GetLastError())
+                self._raise_win_err("failed to read %d bytes" % size, err)
 
         nread = wintypes.DWORD()
-        if not self._get_overlapped_result_ex(self.pipe, olap, nread,
-                                              0 if immediate else self.timeout,
-                                              True):
+        if not self._get_overlapped_result_ex(
+            self.pipe, olap, nread, 0 if immediate else self.timeout, True
+        ):
             err = GetLastError()
             CancelIoEx(self.pipe, olap)
 
             if err == WAIT_TIMEOUT:
-                log('GetOverlappedResultEx timedout')
-                raise SocketTimeout('timed out after waiting %dms for read' %
-                                    self.timeout)
+                log("GetOverlappedResultEx timedout")
+                raise SocketTimeout(
+                    "timed out after waiting %dms for read" % self.timeout
+                )
 
-            log('GetOverlappedResultEx reports error %d', err)
-            self._raise_win_err('error while waiting for read', err)
+            log("GetOverlappedResultEx reports error %d", err)
+            self._raise_win_err("error while waiting for read", err)
 
         nread = nread.value
         if nread == 0:
             # Docs say that named pipes return 0 byte when the other end did
             # a zero byte write.  Since we don't ever do that, the only
             # other way this shows up is if the client has gotten in a weird
             # state, so let's bail out
             CancelIoEx(self.pipe, olap)
-            raise IOError('Async read yielded 0 bytes; unpossible!')
+            raise IOError("Async read yielded 0 bytes; unpossible!")
 
         # Holds precisely the bytes that we read from the prior request
         buf = buf[:nread]
 
         returned_size = min(nread, size)
         if returned_size == nread:
             return buf
 
         # keep any left-overs around for a later read to consume
         self._iobuf = buf[returned_size:]
         return buf[:returned_size]
 
     def write(self, data):
         olap = OVERLAPPED()
         olap.hEvent = self._waitable
 
-        immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data),
-                              None, olap)
+        immediate = WriteFile(
+            self.pipe, ctypes.c_char_p(data), len(data), None, olap
+        )
 
         if not immediate:
             err = GetLastError()
             if err != ERROR_IO_PENDING:
-                self._raise_win_err('failed to write %d bytes' % len(data),
-                                    GetLastError())
+                self._raise_win_err(
+                    "failed to write %d bytes to handle %r"
+                    % (len(data), self.pipe),
+                    err,
+                )
 
         # Obtain results, waiting if needed
         nwrote = wintypes.DWORD()
-        if self._get_overlapped_result_ex(self.pipe, olap, nwrote,
-                                          0 if immediate else self.timeout,
-                                          True):
-            log('made write of %d bytes', nwrote.value)
+        if self._get_overlapped_result_ex(
+            self.pipe, olap, nwrote, 0 if immediate else self.timeout, True
+        ):
+            log("made write of %d bytes", nwrote.value)
             return nwrote.value
 
         err = GetLastError()
 
         # It's potentially unsafe to allow the write to continue after
         # we unwind, so let's make a best effort to avoid that happening
         CancelIoEx(self.pipe, olap)
 
         if err == WAIT_TIMEOUT:
-            raise SocketTimeout('timed out after waiting %dms for write' %
-                                self.timeout)
-        self._raise_win_err('error while waiting for write of %d bytes' %
-                            len(data), err)
+            raise SocketTimeout(
+                "timed out after waiting %dms for write" % self.timeout
+            )
+        self._raise_win_err(
+            "error while waiting for write of %d bytes" % len(data), err
+        )
+
+
+def _default_binpath(binpath=None):
+    if binpath:
+        return binpath
+    # The test harness sets WATCHMAN_BINARY to the binary under test,
+    # so we use that by default, otherwise, allow resolving watchman
+    # from the users PATH.
+    return os.environ.get("WATCHMAN_BINARY", "watchman")
 
 
 class CLIProcessTransport(Transport):
     """ open a pipe to the cli to talk to the service
     This intended to be used only in the test harness!
 
     The CLI is an oddball because we only support JSON input
     and cannot send multiple commands through the same instance,
     so we spawn a new process for each command.
 
     We disable server spawning for this implementation, again, because
     it is intended to be used only in our test harness.  You really
     should not need to use the CLI transport for anything real.
 
     While the CLI can output in BSER, our Transport interface doesn't
     support telling this instance that it should do so.  That effectively
     limits this implementation to JSON input and output only at this time.
 
     It is the responsibility of the caller to set the send and
     receive codecs appropriately.
     """
+
     proc = None
     closed = True
 
-    def __init__(self, sockpath, timeout, watchman_exe):
+    def __init__(self, sockpath, timeout, binpath=None):
         self.sockpath = sockpath
         self.timeout = timeout
-        self.watchman_exe = watchman_exe
+        self.binpath = _default_binpath(binpath)
 
     def close(self):
         if self.proc:
             if self.proc.pid is not None:
                 self.proc.kill()
             self.proc.stdin.close()
             self.proc.stdout.close()
+            self.proc.wait()
             self.proc = None
 
     def _connect(self):
         if self.proc:
             return self.proc
         args = [
-            self.watchman_exe,
-            '--sockname={0}'.format(self.sockpath),
-            '--logfile=/BOGUS',
-            '--statefile=/BOGUS',
-            '--no-spawn',
-            '--no-local',
-            '--no-pretty',
-            '-j',
+            self.binpath,
+            "--sockname={0}".format(self.sockpath),
+            "--logfile=/BOGUS",
+            "--statefile=/BOGUS",
+            "--no-spawn",
+            "--no-local",
+            "--no-pretty",
+            "-j",
         ]
-        self.proc = subprocess.Popen(pycompat.rapply(procutil.tonativestr,
-                                                     args),
-                                     stdin=subprocess.PIPE,
-                                     stdout=subprocess.PIPE)
+        self.proc = subprocess.Popen(
+            args, stdin=subprocess.PIPE, stdout=subprocess.PIPE
+        )
         return self.proc
 
     def readBytes(self, size):
         self._connect()
         res = self.proc.stdout.read(size)
-        if res == '':
-            raise WatchmanError('EOF on CLI process transport')
+        if not res:
+            raise WatchmanError("EOF on CLI process transport")
         return res
 
     def write(self, data):
         if self.closed:
             self.close()
             self.closed = False
         self._connect()
         res = self.proc.stdin.write(data)
         self.proc.stdin.close()
         self.closed = True
         return res
 
 
 class BserCodec(Codec):
     """ use the BSER encoding.  This is the default, preferred codec """
 
+    def __init__(self, transport, value_encoding, value_errors):
+        super(BserCodec, self).__init__(transport)
+        self._value_encoding = value_encoding
+        self._value_errors = value_errors
+
     def _loads(self, response):
-        return bser.loads(response) # Defaults to BSER v1
+        return bser.loads(
+            response,
+            value_encoding=self._value_encoding,
+            value_errors=self._value_errors,
+        )
 
     def receive(self):
         buf = [self.transport.readBytes(sniff_len)]
         if not buf[0]:
-            raise WatchmanError('empty watchman response')
+            raise WatchmanError("empty watchman response")
 
         _1, _2, elen = bser.pdu_info(buf[0])
 
         rlen = len(buf[0])
         while elen > rlen:
             buf.append(self.transport.readBytes(elen - rlen))
             rlen += len(buf[-1])
 
-        response = b''.join(buf)
+        response = b"".join(buf)
         try:
             res = self._loads(response)
             return res
         except ValueError as e:
-            raise WatchmanError('watchman response decode error: %s' % e)
+            raise WatchmanError("watchman response decode error: %s" % e)
 
     def send(self, *args):
-        cmd = bser.dumps(*args) # Defaults to BSER v1
+        cmd = bser.dumps(*args)  # Defaults to BSER v1
         self.transport.write(cmd)
 
 
 class ImmutableBserCodec(BserCodec):
     """ use the BSER encoding, decoding values using the newer
         immutable object support """
 
     def _loads(self, response):
-        return bser.loads(response, False) # Defaults to BSER v1
+        return bser.loads(
+            response,
+            False,
+            value_encoding=self._value_encoding,
+            value_errors=self._value_errors,
+        )
 
 
 class Bser2WithFallbackCodec(BserCodec):
     """ use BSER v2 encoding """
 
-    def __init__(self, transport):
-        super(Bser2WithFallbackCodec, self).__init__(transport)
-        # Once the server advertises support for bser-v2 we should switch this
-        # to 'required' on Python 3.
-        self.send(["version", {"optional": ["bser-v2"]}])
+    def __init__(self, transport, value_encoding, value_errors):
+        super(Bser2WithFallbackCodec, self).__init__(
+            transport, value_encoding, value_errors
+        )
+        if compat.PYTHON3:
+            bserv2_key = "required"
+        else:
+            bserv2_key = "optional"
+
+        self.send(["version", {bserv2_key: ["bser-v2"]}])
 
         capabilities = self.receive()
 
-        if 'error' in capabilities:
-          raise Exception('Unsupported BSER version')
+        if "error" in capabilities:
+            raise BSERv1Unsupported(
+                "The watchman server version does not support Python 3. Please "
+                "upgrade your watchman server."
+            )
 
-        if capabilities['capabilities']['bser-v2']:
+        if capabilities["capabilities"]["bser-v2"]:
             self.bser_version = 2
             self.bser_capabilities = 0
         else:
             self.bser_version = 1
             self.bser_capabilities = 0
 
-    def _loads(self, response):
-        return bser.loads(response)
-
     def receive(self):
         buf = [self.transport.readBytes(sniff_len)]
         if not buf[0]:
-            raise WatchmanError('empty watchman response')
+            raise WatchmanError("empty watchman response")
 
         recv_bser_version, recv_bser_capabilities, elen = bser.pdu_info(buf[0])
 
-        if hasattr(self, 'bser_version'):
-          # Readjust BSER version and capabilities if necessary
-          self.bser_version = max(self.bser_version, recv_bser_version)
-          self.capabilities = self.bser_capabilities & recv_bser_capabilities
+        if hasattr(self, "bser_version"):
+            # Readjust BSER version and capabilities if necessary
+            self.bser_version = max(self.bser_version, recv_bser_version)
+            self.capabilities = self.bser_capabilities & recv_bser_capabilities
 
         rlen = len(buf[0])
         while elen > rlen:
             buf.append(self.transport.readBytes(elen - rlen))
             rlen += len(buf[-1])
 
-        response = b''.join(buf)
+        response = b"".join(buf)
         try:
             res = self._loads(response)
             return res
         except ValueError as e:
-            raise WatchmanError('watchman response decode error: %s' % e)
+            raise WatchmanError("watchman response decode error: %s" % e)
 
     def send(self, *args):
-        if hasattr(self, 'bser_version'):
-            cmd = bser.dumps(*args, version=self.bser_version,
-                capabilities=self.bser_capabilities)
+        if hasattr(self, "bser_version"):
+            cmd = bser.dumps(
+                *args,
+                version=self.bser_version,
+                capabilities=self.bser_capabilities
+            )
         else:
             cmd = bser.dumps(*args)
         self.transport.write(cmd)
 
 
+class ImmutableBser2Codec(Bser2WithFallbackCodec, ImmutableBserCodec):
+    """ use the BSER encoding, decoding values using the newer
+        immutable object support """
+
+    pass
+
+
 class JsonCodec(Codec):
     """ Use json codec.  This is here primarily for testing purposes """
+
     json = None
 
     def __init__(self, transport):
         super(JsonCodec, self).__init__(transport)
         # optional dep on json, only if JsonCodec is used
         import json
+
         self.json = json
 
     def receive(self):
         line = self.transport.readLine()
         try:
             # In Python 3, json.loads is a transformation from Unicode string to
             # objects possibly containing Unicode strings. We typically expect
             # the JSON blob to be ASCII-only with non-ASCII characters escaped,
             # but it's possible we might get non-ASCII bytes that are valid
             # UTF-8.
             if compat.PYTHON3:
-                line = line.decode('utf-8')
+                line = line.decode("utf-8")
             return self.json.loads(line)
         except Exception as e:
             print(e, line)
             raise
 
     def send(self, *args):
         cmd = self.json.dumps(*args)
         # In Python 3, json.dumps is a transformation from objects possibly
         # containing Unicode strings to Unicode string. Even with (the default)
         # ensure_ascii=True, dumps returns a Unicode string.
         if compat.PYTHON3:
-            cmd = cmd.encode('ascii')
+            cmd = cmd.encode("ascii")
         self.transport.write(cmd + b"\n")
 
 
 class client(object):
     """ Handles the communication with the watchman service """
+
     sockpath = None
     transport = None
     sendCodec = None
     recvCodec = None
     sendConn = None
     recvConn = None
     subs = {}  # Keyed by subscription name
     sub_by_root = {}  # Keyed by root, then by subscription name
     logs = []  # When log level is raised
-    unilateral = ['log', 'subscription']
+    unilateral = ["log", "subscription"]
     tport = None
     useImmutableBser = None
-    watchman_exe = None
+    pid = None
 
-    def __init__(self,
-                 sockpath=None,
-                 timeout=1.0,
-                 transport=None,
-                 sendEncoding=None,
-                 recvEncoding=None,
-                 useImmutableBser=False,
-                 watchman_exe=None):
+    def __init__(
+        self,
+        sockpath=None,
+        timeout=1.0,
+        transport=None,
+        sendEncoding=None,
+        recvEncoding=None,
+        useImmutableBser=False,
+        # use False for these two because None has a special
+        # meaning
+        valueEncoding=False,
+        valueErrors=False,
+        binpath=None,
+    ):
         self.sockpath = sockpath
         self.timeout = timeout
         self.useImmutableBser = useImmutableBser
-        self.watchman_exe = watchman_exe
+        self.binpath = _default_binpath(binpath)
 
         if inspect.isclass(transport) and issubclass(transport, Transport):
             self.transport = transport
         else:
-            transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local'
-            if transport == 'local' and os.name == 'nt':
+            transport = transport or os.getenv("WATCHMAN_TRANSPORT") or "local"
+            if transport == "local" and os.name == "nt":
                 self.transport = WindowsNamedPipeTransport
-            elif transport == 'local':
+            elif transport == "local":
                 self.transport = UnixSocketTransport
-            elif transport == 'cli':
+            elif transport == "cli":
                 self.transport = CLIProcessTransport
                 if sendEncoding is None:
-                    sendEncoding = 'json'
+                    sendEncoding = "json"
                 if recvEncoding is None:
                     recvEncoding = sendEncoding
             else:
-                raise WatchmanError('invalid transport %s' % transport)
+                raise WatchmanError("invalid transport %s" % transport)
 
-        sendEncoding = str(sendEncoding or os.getenv('WATCHMAN_ENCODING') or
-                           'bser')
-        recvEncoding = str(recvEncoding or os.getenv('WATCHMAN_ENCODING') or
-                           'bser')
+        sendEncoding = str(
+            sendEncoding or os.getenv("WATCHMAN_ENCODING") or "bser"
+        )
+        recvEncoding = str(
+            recvEncoding or os.getenv("WATCHMAN_ENCODING") or "bser"
+        )
 
         self.recvCodec = self._parseEncoding(recvEncoding)
         self.sendCodec = self._parseEncoding(sendEncoding)
 
+        # We want to act like the native OS methods as much as possible. This
+        # means returning bytestrings on Python 2 by default and Unicode
+        # strings on Python 3. However we take an optional argument that lets
+        # users override this.
+        if valueEncoding is False:
+            if compat.PYTHON3:
+                self.valueEncoding = encoding.get_local_encoding()
+                self.valueErrors = encoding.default_local_errors
+            else:
+                self.valueEncoding = None
+                self.valueErrors = None
+        else:
+            self.valueEncoding = valueEncoding
+            if valueErrors is False:
+                self.valueErrors = encoding.default_local_errors
+            else:
+                self.valueErrors = valueErrors
+
+    def _makeBSERCodec(self, codec):
+        def make_codec(transport):
+            return codec(transport, self.valueEncoding, self.valueErrors)
+
+        return make_codec
+
     def _parseEncoding(self, enc):
-        if enc == 'bser':
+        if enc == "bser":
             if self.useImmutableBser:
-                return ImmutableBserCodec
-            return BserCodec
-        elif enc == 'experimental-bser-v2':
-          return Bser2WithFallbackCodec
-        elif enc == 'json':
+                return self._makeBSERCodec(ImmutableBser2Codec)
+            return self._makeBSERCodec(Bser2WithFallbackCodec)
+        elif enc == "bser-v1":
+            if compat.PYTHON3:
+                raise BSERv1Unsupported(
+                    "Python 3 does not support the BSER v1 encoding: specify "
+                    '"bser" or omit the sendEncoding and recvEncoding '
+                    "arguments"
+                )
+            if self.useImmutableBser:
+                return self._makeBSERCodec(ImmutableBserCodec)
+            return self._makeBSERCodec(BserCodec)
+        elif enc == "json":
             return JsonCodec
         else:
-            raise WatchmanError('invalid encoding %s' % enc)
+            raise WatchmanError("invalid encoding %s" % enc)
 
     def _hasprop(self, result, name):
         if self.useImmutableBser:
             return hasattr(result, name)
         return name in result
 
     def _resolvesockname(self):
         # if invoked via a trigger, watchman will set this env var; we
         # should use it unless explicitly set otherwise
-        path = os.getenv('WATCHMAN_SOCK')
+        path = os.getenv("WATCHMAN_SOCK")
         if path:
             return path
 
-        cmd = [self.watchman_exe, '--output-encoding=bser', 'get-sockname']
+        cmd = [self.binpath, "--output-encoding=bser", "get-sockname"]
         try:
-            args = dict(stdout=subprocess.PIPE,
-                        stderr=subprocess.PIPE,
-                        close_fds=os.name != 'nt')
+            args = dict(
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE
+            )  # noqa: C408
 
-            if os.name == 'nt':
+            if os.name == "nt":
                 # if invoked via an application with graphical user interface,
                 # this call will cause a brief command window pop-up.
                 # Using the flag STARTF_USESHOWWINDOW to avoid this behavior.
                 startupinfo = subprocess.STARTUPINFO()
                 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
-                args['startupinfo'] = startupinfo
+                args["startupinfo"] = startupinfo
 
-            p = subprocess.Popen(pycompat.rapply(procutil.tonativestr, cmd),
-                                 **args)
+            p = subprocess.Popen(cmd, **args)
 
         except OSError as e:
-            raise WatchmanError('"watchman" executable not in PATH (%s)' % e)
+            raise WatchmanError('"watchman" executable not in PATH (%s)', e)
 
         stdout, stderr = p.communicate()
         exitcode = p.poll()
 
         if exitcode:
             raise WatchmanError("watchman exited with code %d" % exitcode)
 
         result = bser.loads(stdout)
-        if b'error' in result:
-            raise WatchmanError('get-sockname error: %s' % result['error'])
+        if "error" in result:
+            raise WatchmanError("get-sockname error: %s" % result["error"])
 
-        return result[b'sockname']
+        return result["sockname"]
 
     def _connect(self):
         """ establish transport connection """
 
         if self.recvConn:
+            if self.pid != os.getpid():
+                raise UseAfterFork(
+                    "do not re-use a connection after fork; open a new client instead"
+                )
             return
 
         if self.sockpath is None:
             self.sockpath = self._resolvesockname()
 
-        self.tport = self.transport(self.sockpath, self.timeout, self.watchman_exe)
+        kwargs = {}
+        if self.transport == CLIProcessTransport:
+            kwargs["binpath"] = self.binpath
+
+        self.tport = self.transport(self.sockpath, self.timeout, **kwargs)
         self.sendConn = self.sendCodec(self.tport)
         self.recvConn = self.recvCodec(self.tport)
+        self.pid = os.getpid()
 
     def __del__(self):
         self.close()
 
+    def __enter__(self):
+        self._connect()
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        self.close()
+
     def close(self):
         if self.tport:
             self.tport.close()
             self.tport = None
             self.recvConn = None
             self.sendConn = None
 
     def receive(self):
         """ receive the next PDU from the watchman service
 
         If the client has activated subscriptions or logs then
         this PDU may be a unilateral PDU sent by the service to
         inform the client of a log event or subscription change.
 
         It may also simply be the response portion of a request
         initiated by query.
 
         There are clients in production that subscribe and call
         this in a loop to retrieve all subscription responses,
         so care should be taken when making changes here.
         """
 
         self._connect()
         result = self.recvConn.receive()
-        if self._hasprop(result, 'error'):
-            error = result['error']
-            if compat.PYTHON3 and isinstance(self.recvConn, BserCodec):
-                error = result['error'].decode('utf-8', 'surrogateescape')
-            raise CommandError(error)
+        if self._hasprop(result, "error"):
+            raise CommandError(result["error"])
 
-        if self._hasprop(result, 'log'):
-            log = result['log']
-            if compat.PYTHON3 and isinstance(self.recvConn, BserCodec):
-                log = log.decode('utf-8', 'surrogateescape')
-            self.logs.append(log)
+        if self._hasprop(result, "log"):
+            self.logs.append(result["log"])
 
-        if self._hasprop(result, 'subscription'):
-            sub = result['subscription']
+        if self._hasprop(result, "subscription"):
+            sub = result["subscription"]
             if not (sub in self.subs):
                 self.subs[sub] = []
             self.subs[sub].append(result)
 
             # also accumulate in {root,sub} keyed store
-            root = os.path.normcase(result['root'])
+            root = os.path.normpath(os.path.normcase(result["root"]))
             if not root in self.sub_by_root:
                 self.sub_by_root[root] = {}
             if not sub in self.sub_by_root[root]:
                 self.sub_by_root[root][sub] = []
             self.sub_by_root[root][sub].append(result)
 
         return result
 
     def isUnilateralResponse(self, res):
-        if 'unilateral' in res and res['unilateral']:
+        if "unilateral" in res and res["unilateral"]:
             return True
         # Fall back to checking for known unilateral responses
         for k in self.unilateral:
             if k in res:
                 return True
         return False
 
     def getLog(self, remove=True):
         """ Retrieve buffered log data
 
         If remove is true the data will be removed from the buffer.
         Otherwise it will be left in the buffer
         """
         res = self.logs
         if remove:
             self.logs = []
         return res
 
     def getSubscription(self, name, remove=True, root=None):
         """ Retrieve the data associated with a named subscription
 
         If remove is True (the default), the subscription data is removed
         from the buffer.  Otherwise the data is returned but left in
         the buffer.
 
         Returns None if there is no data associated with `name`
 
         If root is not None, then only return the subscription
         data that matches both root and name.  When used in this way,
         remove processing impacts both the unscoped and scoped stores
         for the subscription data.
         """
-        if compat.PYTHON3 and issubclass(self.recvCodec, BserCodec):
-            # People may pass in Unicode strings here -- but currently BSER only
-            # returns bytestrings. Deal with that.
-            if isinstance(root, str):
-                root = encoding.encode_local(root)
-            if isinstance(name, str):
-                name = name.encode('utf-8')
-
         if root is not None:
-            if not root in self.sub_by_root:
+            root = os.path.normpath(os.path.normcase(root))
+            if root not in self.sub_by_root:
                 return None
-            if not name in self.sub_by_root[root]:
+            if name not in self.sub_by_root[root]:
                 return None
             sub = self.sub_by_root[root][name]
             if remove:
                 del self.sub_by_root[root][name]
                 # don't let this grow unbounded
                 if name in self.subs:
                     del self.subs[name]
             return sub
 
-        if not (name in self.subs):
+        if name not in self.subs:
             return None
         sub = self.subs[name]
         if remove:
             del self.subs[name]
         return sub
 
     def query(self, *args):
         """ Send a query to the watchman service and return the response
 
         This call will block until the response is returned.
         If any unilateral responses are sent by the service in between
         the request-response they will be buffered up in the client object
         and NOT returned via this method.
         """
 
-        log('calling client.query')
+        log("calling client.query")
         self._connect()
         try:
             self.sendConn.send(args)
 
             res = self.receive()
             while self.isUnilateralResponse(res):
                 res = self.receive()
 
             return res
         except EnvironmentError as ee:
             # When we can depend on Python 3, we can use PEP 3134
             # exception chaining here.
             raise WatchmanEnvironmentError(
-                'I/O error communicating with watchman daemon',
+                "I/O error communicating with watchman daemon",
                 ee.errno,
                 ee.strerror,
-                args)
+                args,
+            )
         except WatchmanError as ex:
             ex.setCommand(args)
             raise
 
     def capabilityCheck(self, optional=None, required=None):
         """ Perform a server capability check """
-        res = self.query('version', {
-            'optional': optional or [],
-            'required': required or []
-        })
+        res = self.query(
+            "version", {"optional": optional or [], "required": required or []}
+        )
 
-        if not self._hasprop(res, 'capabilities'):
+        if not self._hasprop(res, "capabilities"):
             # Server doesn't support capabilities, so we need to
             # synthesize the results based on the version
             capabilities.synthesize(res, optional)
-            if 'error' in res:
-                raise CommandError(res['error'])
+            if "error" in res:
+                raise CommandError(res["error"])
 
         return res
 
     def setTimeout(self, value):
         self.recvConn.setTimeout(value)
         self.sendConn.setTimeout(value)

hgext/fsmonitor/pywatchman/bser.c

@@ -1,1223 +1,1241 b''
 /*
 Copyright (c) 2013-2015, Facebook, Inc.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 
  * Redistributions of source code must retain the above copyright notice,
    this list of conditions and the following disclaimer.
 
  * Redistributions in binary form must reproduce the above copyright notice,
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.
 
  * Neither the name Facebook nor the names of its contributors may be used to
    endorse or promote products derived from this software without specific
    prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
 #include <Python.h>
 #include <bytesobject.h>
 #ifdef _MSC_VER
 #define inline __inline
 #if _MSC_VER >= 1800
 #include <stdint.h>
 #else
 // The compiler associated with Python 2.7 on Windows doesn't ship
 // with stdint.h, so define the small subset that we use here.
 typedef __int8 int8_t;
 typedef __int16 int16_t;
 typedef __int32 int32_t;
 typedef __int64 int64_t;
 typedef unsigned __int8 uint8_t;
 typedef unsigned __int16 uint16_t;
 typedef unsigned __int32 uint32_t;
 typedef unsigned __int64 uint64_t;
 #define UINT32_MAX 4294967295U
 #endif
 #endif
 
 // clang-format off
 /* Return the smallest size int that can store the value */
 #define INT_SIZE(x) (((x) == ((int8_t)x))  ? 1 :    \
                      ((x) == ((int16_t)x)) ? 2 :    \
                      ((x) == ((int32_t)x)) ? 4 : 8)
 
 #define BSER_ARRAY     0x00
 #define BSER_OBJECT    0x01
 #define BSER_BYTESTRING 0x02
 #define BSER_INT8      0x03
 #define BSER_INT16     0x04
 #define BSER_INT32     0x05
 #define BSER_INT64     0x06
 #define BSER_REAL      0x07
 #define BSER_TRUE      0x08
 #define BSER_FALSE     0x09
 #define BSER_NULL      0x0a
 #define BSER_TEMPLATE  0x0b
 #define BSER_SKIP      0x0c
 #define BSER_UTF8STRING 0x0d
 // clang-format on
 
 // An immutable object representation of BSER_OBJECT.
 // Rather than build a hash table, key -> value are obtained
 // by walking the list of keys to determine the offset into
 // the values array.  The assumption is that the number of
 // array elements will be typically small (~6 for the top
 // level query result and typically 3-5 for the file entries)
 // so that the time overhead for this is small compared to
 // using a proper hash table.  Even with this simplistic
 // approach, this is still faster for the mercurial use case
 // as it helps to eliminate creating N other objects to
 // represent the stat information in the hgwatchman extension
 // clang-format off
 typedef struct {
   PyObject_HEAD
   PyObject *keys;   // tuple of field names
   PyObject *values; // tuple of values
 } bserObject;
 // clang-format on
 
 static Py_ssize_t bserobj_tuple_length(PyObject* o) {
   bserObject* obj = (bserObject*)o;
 
   return PySequence_Length(obj->keys);
 }
 
 static PyObject* bserobj_tuple_item(PyObject* o, Py_ssize_t i) {
   bserObject* obj = (bserObject*)o;
 
   return PySequence_GetItem(obj->values, i);
 }
 
 // clang-format off
 static PySequenceMethods bserobj_sq = {
   bserobj_tuple_length,      /* sq_length */
   0,                         /* sq_concat */
   0,                         /* sq_repeat */
   bserobj_tuple_item,        /* sq_item */
   0,                         /* sq_ass_item */
   0,                         /* sq_contains */
   0,                         /* sq_inplace_concat */
   0                          /* sq_inplace_repeat */
 };
 // clang-format on
 
 static void bserobj_dealloc(PyObject* o) {
   bserObject* obj = (bserObject*)o;
 
   Py_CLEAR(obj->keys);
   Py_CLEAR(obj->values);
   PyObject_Del(o);
 }
 
 static PyObject* bserobj_getattrro(PyObject* o, PyObject* name) {
   bserObject* obj = (bserObject*)o;
   Py_ssize_t i, n;
   PyObject* name_bytes = NULL;
   PyObject* ret = NULL;
-  const char* namestr = NULL;
+  const char* namestr;
 
   if (PyIndex_Check(name)) {
     i = PyNumber_AsSsize_t(name, PyExc_IndexError);
     if (i == -1 && PyErr_Occurred()) {
       goto bail;
     }
+    ret = PySequence_GetItem(obj->values, i);
+    goto bail;
+  }
 
-    if (i == 8 && PySequence_Size(obj->values) < 9) {
-      // Hack alert: Python 3 removed support for os.stat().st_mtime
-      // being an integer.Instead, if you need an integer, you have to
-      // use os.stat()[stat.ST_MTIME] instead. stat.ST_MTIME is 8, and
-      // our stat tuples are shorter than that, so we can detect
-      // requests for index 8 on tuples shorter than that and return
-      // st_mtime instead.
-      namestr = "st_mtime";
-    } else {
-      ret = PySequence_GetItem(obj->values, i);
+  // We can be passed in Unicode objects here -- we don't support anything other
+  // than UTF-8 for keys.
+  if (PyUnicode_Check(name)) {
+    name_bytes = PyUnicode_AsUTF8String(name);
+    if (name_bytes == NULL) {
       goto bail;
     }
+    namestr = PyBytes_AsString(name_bytes);
   } else {
-    // We can be passed in Unicode objects here -- we don't support anything other
-    // than UTF-8 for keys.
-    if (PyUnicode_Check(name)) {
-      name_bytes = PyUnicode_AsUTF8String(name);
-      if (name_bytes == NULL) {
-        goto bail;
-      }
-      namestr = PyBytes_AsString(name_bytes);
-    } else {
-      namestr = PyBytes_AsString(name);
-    }
+    namestr = PyBytes_AsString(name);
   }
 
   if (namestr == NULL) {
     goto bail;
   }
   // hack^Wfeature to allow mercurial to use "st_size" to reference "size"
   if (!strncmp(namestr, "st_", 3)) {
     namestr += 3;
   }
 
   n = PyTuple_GET_SIZE(obj->keys);
   for (i = 0; i < n; i++) {
     const char* item_name = NULL;
     PyObject* key = PyTuple_GET_ITEM(obj->keys, i);
 
     item_name = PyBytes_AsString(key);
     if (!strcmp(item_name, namestr)) {
       ret = PySequence_GetItem(obj->values, i);
       goto bail;
     }
   }
 
   PyErr_Format(
       PyExc_AttributeError, "bserobject has no attribute '%.400s'", namestr);
 bail:
   Py_XDECREF(name_bytes);
   return ret;
 }
 
 // clang-format off
 static PyMappingMethods bserobj_map = {
   bserobj_tuple_length,     /* mp_length */
   bserobj_getattrro,        /* mp_subscript */
   0                         /* mp_ass_subscript */
 };
 
 PyTypeObject bserObjectType = {
   PyVarObject_HEAD_INIT(NULL, 0)
   "bserobj_tuple",           /* tp_name */
   sizeof(bserObject),        /* tp_basicsize */
   0,                         /* tp_itemsize */
   bserobj_dealloc,           /* tp_dealloc */
   0,                         /* tp_print */
   0,                         /* tp_getattr */
   0,                         /* tp_setattr */
   0,                         /* tp_compare */
   0,                         /* tp_repr */
   0,                         /* tp_as_number */
   &bserobj_sq,               /* tp_as_sequence */
   &bserobj_map,              /* tp_as_mapping */
   0,                         /* tp_hash  */
   0,                         /* tp_call */
   0,                         /* tp_str */
   bserobj_getattrro,         /* tp_getattro */
   0,                         /* tp_setattro */
   0,                         /* tp_as_buffer */
   Py_TPFLAGS_DEFAULT,        /* tp_flags */
   "bserobj tuple",           /* tp_doc */
   0,                         /* tp_traverse */
   0,                         /* tp_clear */
   0,                         /* tp_richcompare */
   0,                         /* tp_weaklistoffset */
   0,                         /* tp_iter */
   0,                         /* tp_iternext */
   0,                         /* tp_methods */
   0,                         /* tp_members */
   0,                         /* tp_getset */
   0,                         /* tp_base */
   0,                         /* tp_dict */
   0,                         /* tp_descr_get */
   0,                         /* tp_descr_set */
   0,                         /* tp_dictoffset */
   0,                         /* tp_init */
   0,                         /* tp_alloc */
   0,                         /* tp_new */
 };
 // clang-format on
 
 typedef struct loads_ctx {
   int mutable;
   const char* value_encoding;
   const char* value_errors;
   uint32_t bser_version;
   uint32_t bser_capabilities;
 } unser_ctx_t;
 
 static PyObject*
 bser_loads_recursive(const char** ptr, const char* end, const unser_ctx_t* ctx);
 
 static const char bser_true = BSER_TRUE;
 static const char bser_false = BSER_FALSE;
 static const char bser_null = BSER_NULL;
 static const char bser_bytestring_hdr = BSER_BYTESTRING;
 static const char bser_array_hdr = BSER_ARRAY;
 static const char bser_object_hdr = BSER_OBJECT;
 
 static inline uint32_t next_power_2(uint32_t n) {
   n |= (n >> 16);
   n |= (n >> 8);
   n |= (n >> 4);
   n |= (n >> 2);
   n |= (n >> 1);
   return n + 1;
 }
 
 // A buffer we use for building up the serialized result
 struct bser_buffer {
   char* buf;
   int wpos, allocd;
   uint32_t bser_version;
   uint32_t capabilities;
 };
 typedef struct bser_buffer bser_t;
 
 static int bser_append(bser_t* bser, const char* data, uint32_t len) {
   int newlen = next_power_2(bser->wpos + len);
   if (newlen > bser->allocd) {
     char* nbuf = realloc(bser->buf, newlen);
     if (!nbuf) {
       return 0;
     }
 
     bser->buf = nbuf;
     bser->allocd = newlen;
   }
 
   memcpy(bser->buf + bser->wpos, data, len);
   bser->wpos += len;
   return 1;
 }
 
 static int bser_init(bser_t* bser, uint32_t version, uint32_t capabilities) {
   bser->allocd = 8192;
   bser->wpos = 0;
   bser->buf = malloc(bser->allocd);
   bser->bser_version = version;
   bser->capabilities = capabilities;
   if (!bser->buf) {
     return 0;
   }
 
 // Leave room for the serialization header, which includes
 // our overall length.  To make things simpler, we'll use an
 // int32 for the header
 #define EMPTY_HEADER "\x00\x01\x05\x00\x00\x00\x00"
 
 // Version 2 also carries an integer indicating the capabilities. The
 // capabilities integer comes before the PDU size.
 #define EMPTY_HEADER_V2 "\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00"
   if (version == 2) {
     bser_append(bser, EMPTY_HEADER_V2, sizeof(EMPTY_HEADER_V2) - 1);
   } else {
     bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER) - 1);
   }
 
   return 1;
 }
 
 static void bser_dtor(bser_t* bser) {
   free(bser->buf);
   bser->buf = NULL;
 }
 
 static int bser_long(bser_t* bser, int64_t val) {
   int8_t i8;
   int16_t i16;
   int32_t i32;
   int64_t i64;
   char sz;
   int size = INT_SIZE(val);
   char* iptr;
 
   switch (size) {
     case 1:
       sz = BSER_INT8;
       i8 = (int8_t)val;
       iptr = (char*)&i8;
       break;
     case 2:
       sz = BSER_INT16;
       i16 = (int16_t)val;
       iptr = (char*)&i16;
       break;
     case 4:
       sz = BSER_INT32;
       i32 = (int32_t)val;
       iptr = (char*)&i32;
       break;
     case 8:
       sz = BSER_INT64;
       i64 = (int64_t)val;
       iptr = (char*)&i64;
       break;
     default:
       PyErr_SetString(PyExc_RuntimeError, "Cannot represent this long value!?");
       return 0;
   }
 
   if (!bser_append(bser, &sz, sizeof(sz))) {
     return 0;
   }
 
   return bser_append(bser, iptr, size);
 }
 
 static int bser_bytestring(bser_t* bser, PyObject* sval) {
   char* buf = NULL;
   Py_ssize_t len;
   int res;
   PyObject* utf = NULL;
 
   if (PyUnicode_Check(sval)) {
     utf = PyUnicode_AsEncodedString(sval, "utf-8", "ignore");
     sval = utf;
   }
 
   res = PyBytes_AsStringAndSize(sval, &buf, &len);
   if (res == -1) {
     res = 0;
     goto out;
   }
 
   if (!bser_append(bser, &bser_bytestring_hdr, sizeof(bser_bytestring_hdr))) {
     res = 0;
     goto out;
   }
 
   if (!bser_long(bser, len)) {
     res = 0;
     goto out;
   }
 
   if (len > UINT32_MAX) {
     PyErr_Format(PyExc_ValueError, "string too big");
     res = 0;
     goto out;
   }
 
   res = bser_append(bser, buf, (uint32_t)len);
 
 out:
   if (utf) {
     Py_DECREF(utf);
   }
 
   return res;
 }
 
 static int bser_recursive(bser_t* bser, PyObject* val) {
   if (PyBool_Check(val)) {
     if (val == Py_True) {
       return bser_append(bser, &bser_true, sizeof(bser_true));
     }
     return bser_append(bser, &bser_false, sizeof(bser_false));
   }
 
   if (val == Py_None) {
     return bser_append(bser, &bser_null, sizeof(bser_null));
   }
 
 // Python 3 has one integer type.
 #if PY_MAJOR_VERSION < 3
   if (PyInt_Check(val)) {
     return bser_long(bser, PyInt_AS_LONG(val));
   }
 #endif // PY_MAJOR_VERSION < 3
 
   if (PyLong_Check(val)) {
     return bser_long(bser, PyLong_AsLongLong(val));
   }
 
   if (PyBytes_Check(val) || PyUnicode_Check(val)) {
     return bser_bytestring(bser, val);
   }
 
   if (PyFloat_Check(val)) {
     double dval = PyFloat_AS_DOUBLE(val);
     char sz = BSER_REAL;
 
     if (!bser_append(bser, &sz, sizeof(sz))) {
       return 0;
     }
 
     return bser_append(bser, (char*)&dval, sizeof(dval));
   }
 
   if (PyList_Check(val)) {
     Py_ssize_t i, len = PyList_GET_SIZE(val);
 
     if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) {
       return 0;
     }
 
     if (!bser_long(bser, len)) {
       return 0;
     }
 
     for (i = 0; i < len; i++) {
       PyObject* ele = PyList_GET_ITEM(val, i);
 
       if (!bser_recursive(bser, ele)) {
         return 0;
       }
     }
 
     return 1;
   }
 
   if (PyTuple_Check(val)) {
     Py_ssize_t i, len = PyTuple_GET_SIZE(val);
 
     if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) {
       return 0;
     }
 
     if (!bser_long(bser, len)) {
       return 0;
     }
 
     for (i = 0; i < len; i++) {
       PyObject* ele = PyTuple_GET_ITEM(val, i);
 
       if (!bser_recursive(bser, ele)) {
         return 0;
       }
     }
 
     return 1;
   }
 
   if (PyMapping_Check(val)) {
     Py_ssize_t len = PyMapping_Length(val);
     Py_ssize_t pos = 0;
     PyObject *key, *ele;
 
     if (!bser_append(bser, &bser_object_hdr, sizeof(bser_object_hdr))) {
       return 0;
     }
 
     if (!bser_long(bser, len)) {
       return 0;
     }
 
     while (PyDict_Next(val, &pos, &key, &ele)) {
       if (!bser_bytestring(bser, key)) {
         return 0;
       }
       if (!bser_recursive(bser, ele)) {
         return 0;
       }
     }
 
     return 1;
   }
 
   PyErr_SetString(PyExc_ValueError, "Unsupported value type");
   return 0;
 }
 
 static PyObject* bser_dumps(PyObject* self, PyObject* args, PyObject* kw) {
   PyObject *val = NULL, *res;
   bser_t bser;
   uint32_t len, bser_version = 1, bser_capabilities = 0;
 
   static char* kw_list[] = {"val", "version", "capabilities", NULL};
 
   if (!PyArg_ParseTupleAndKeywords(
           args,
           kw,
           "O|ii:dumps",
           kw_list,
           &val,
           &bser_version,
           &bser_capabilities)) {
     return NULL;
   }
 
   if (!bser_init(&bser, bser_version, bser_capabilities)) {
     return PyErr_NoMemory();
   }
 
   if (!bser_recursive(&bser, val)) {
     bser_dtor(&bser);
     if (errno == ENOMEM) {
       return PyErr_NoMemory();
     }
     // otherwise, we've already set the error to something reasonable
     return NULL;
   }
 
   // Now fill in the overall length
   if (bser_version == 1) {
     len = bser.wpos - (sizeof(EMPTY_HEADER) - 1);
     memcpy(bser.buf + 3, &len, sizeof(len));
   } else {
     len = bser.wpos - (sizeof(EMPTY_HEADER_V2) - 1);
     // The BSER capabilities block comes before the PDU length
     memcpy(bser.buf + 2, &bser_capabilities, sizeof(bser_capabilities));
     memcpy(bser.buf + 7, &len, sizeof(len));
   }
 
   res = PyBytes_FromStringAndSize(bser.buf, bser.wpos);
   bser_dtor(&bser);
 
   return res;
 }
 
 int bunser_int(const char** ptr, const char* end, int64_t* val) {
   int needed;
   const char* buf = *ptr;
   int8_t i8;
   int16_t i16;
   int32_t i32;
   int64_t i64;
 
   switch (buf[0]) {
     case BSER_INT8:
       needed = 2;
       break;
     case BSER_INT16:
       needed = 3;
       break;
     case BSER_INT32:
       needed = 5;
       break;
     case BSER_INT64:
       needed = 9;
       break;
     default:
       PyErr_Format(
           PyExc_ValueError, "invalid bser int encoding 0x%02x", buf[0]);
       return 0;
   }
   if (end - buf < needed) {
     PyErr_SetString(PyExc_ValueError, "input buffer to small for int encoding");
     return 0;
   }
   *ptr = buf + needed;
   switch (buf[0]) {
     case BSER_INT8:
       memcpy(&i8, buf + 1, sizeof(i8));
       *val = i8;
       return 1;
     case BSER_INT16:
       memcpy(&i16, buf + 1, sizeof(i16));
       *val = i16;
       return 1;
     case BSER_INT32:
       memcpy(&i32, buf + 1, sizeof(i32));
       *val = i32;
       return 1;
     case BSER_INT64:
       memcpy(&i64, buf + 1, sizeof(i64));
       *val = i64;
       return 1;
     default:
       return 0;
   }
 }
 
 static int bunser_bytestring(
     const char** ptr,
     const char* end,
     const char** start,
     int64_t* len) {
   const char* buf = *ptr;
 
   // skip string marker
   buf++;
   if (!bunser_int(&buf, end, len)) {
     return 0;
   }
 
   if (buf + *len > end) {
     PyErr_Format(PyExc_ValueError, "invalid string length in bser data");
     return 0;
   }
 
   *ptr = buf + *len;
   *start = buf;
   return 1;
 }
 
 static PyObject*
 bunser_array(const char** ptr, const char* end, const unser_ctx_t* ctx) {
   const char* buf = *ptr;
   int64_t nitems, i;
   int mutable = ctx->mutable;
   PyObject* res;
 
   // skip array header
   buf++;
   if (!bunser_int(&buf, end, &nitems)) {
     return 0;
   }
   *ptr = buf;
 
   if (nitems > LONG_MAX) {
     PyErr_Format(PyExc_ValueError, "too many items for python array");
     return NULL;
   }
 
   if (mutable) {
     res = PyList_New((Py_ssize_t)nitems);
   } else {
     res = PyTuple_New((Py_ssize_t)nitems);
   }
 
   for (i = 0; i < nitems; i++) {
     PyObject* ele = bser_loads_recursive(ptr, end, ctx);
 
     if (!ele) {
       Py_DECREF(res);
       return NULL;
     }
 
     if (mutable) {
       PyList_SET_ITEM(res, i, ele);
     } else {
       PyTuple_SET_ITEM(res, i, ele);
     }
     // DECREF(ele) not required as SET_ITEM steals the ref
   }
 
   return res;
 }
 
 static PyObject*
 bunser_object(const char** ptr, const char* end, const unser_ctx_t* ctx) {
   const char* buf = *ptr;
   int64_t nitems, i;
   int mutable = ctx->mutable;
   PyObject* res;
   bserObject* obj;
 
   // skip array header
   buf++;
   if (!bunser_int(&buf, end, &nitems)) {
     return 0;
   }
   *ptr = buf;
 
   if (mutable) {
     res = PyDict_New();
   } else {
     obj = PyObject_New(bserObject, &bserObjectType);
     obj->keys = PyTuple_New((Py_ssize_t)nitems);
     obj->values = PyTuple_New((Py_ssize_t)nitems);
     res = (PyObject*)obj;
   }
 
   for (i = 0; i < nitems; i++) {
     const char* keystr;
     int64_t keylen;
     PyObject* key;
     PyObject* ele;
 
     if (!bunser_bytestring(ptr, end, &keystr, &keylen)) {
       Py_DECREF(res);
       return NULL;
     }
 
     if (keylen > LONG_MAX) {
       PyErr_Format(PyExc_ValueError, "string too big for python");
       Py_DECREF(res);
       return NULL;
     }
 
     if (mutable) {
       // This will interpret the key as UTF-8.
       key = PyUnicode_FromStringAndSize(keystr, (Py_ssize_t)keylen);
     } else {
       // For immutable objects we'll manage key lookups, so we can avoid going
       // through the Unicode APIs. This avoids a potentially expensive and
       // definitely unnecessary conversion to UTF-16 and back for Python 2.
       // TODO: On Python 3 the Unicode APIs are smarter: we might be able to use
       // Unicode keys there without an appreciable performance loss.
       key = PyBytes_FromStringAndSize(keystr, (Py_ssize_t)keylen);
     }
 
     if (!key) {
       Py_DECREF(res);
       return NULL;
     }
 
     ele = bser_loads_recursive(ptr, end, ctx);
 
     if (!ele) {
       Py_DECREF(key);
       Py_DECREF(res);
       return NULL;
     }
 
     if (mutable) {
       PyDict_SetItem(res, key, ele);
       Py_DECREF(key);
       Py_DECREF(ele);
     } else {
       /* PyTuple_SET_ITEM steals ele, key */
       PyTuple_SET_ITEM(obj->values, i, ele);
       PyTuple_SET_ITEM(obj->keys, i, key);
     }
   }
 
   return res;
 }
 
 static PyObject*
 bunser_template(const char** ptr, const char* end, const unser_ctx_t* ctx) {
   const char* buf = *ptr;
   int64_t nitems, i;
   int mutable = ctx->mutable;
   PyObject* arrval;
   PyObject* keys;
   Py_ssize_t numkeys, keyidx;
   unser_ctx_t keys_ctx = {0};
   if (mutable) {
     keys_ctx.mutable = 1;
     // Decode keys as UTF-8 in this case.
     keys_ctx.value_encoding = "utf-8";
     keys_ctx.value_errors = "strict";
   } else {
     // Treat keys as bytestrings in this case -- we'll do Unicode conversions at
     // lookup time.
   }
 
   if (buf[1] != BSER_ARRAY) {
     PyErr_Format(PyExc_ValueError, "Expect ARRAY to follow TEMPLATE");
     return NULL;
   }
 
   // skip header
   buf++;
   *ptr = buf;
 
   // Load template keys.
   // For keys we don't want to do any decoding right now.
   keys = bunser_array(ptr, end, &keys_ctx);
   if (!keys) {
     return NULL;
   }
 
   numkeys = PySequence_Length(keys);
 
   // Load number of array elements
   if (!bunser_int(ptr, end, &nitems)) {
     Py_DECREF(keys);
     return 0;
   }
 
   if (nitems > LONG_MAX) {
     PyErr_Format(PyExc_ValueError, "Too many items for python");
     Py_DECREF(keys);
     return NULL;
   }
 
   arrval = PyList_New((Py_ssize_t)nitems);
   if (!arrval) {
     Py_DECREF(keys);
     return NULL;
   }
 
   for (i = 0; i < nitems; i++) {
     PyObject* dict = NULL;
     bserObject* obj = NULL;
 
     if (mutable) {
       dict = PyDict_New();
     } else {
       obj = PyObject_New(bserObject, &bserObjectType);
       if (obj) {
         obj->keys = keys;
         Py_INCREF(obj->keys);
         obj->values = PyTuple_New(numkeys);
       }
       dict = (PyObject*)obj;
     }
     if (!dict) {
     fail:
       Py_DECREF(keys);
       Py_DECREF(arrval);
       return NULL;
     }
 
     for (keyidx = 0; keyidx < numkeys; keyidx++) {
       PyObject* key;
       PyObject* ele;
 
       if (**ptr == BSER_SKIP) {
         *ptr = *ptr + 1;
         ele = Py_None;
         Py_INCREF(ele);
       } else {
         ele = bser_loads_recursive(ptr, end, ctx);
       }
 
       if (!ele) {
         goto fail;
       }
 
       if (mutable) {
         key = PyList_GET_ITEM(keys, keyidx);
         PyDict_SetItem(dict, key, ele);
         Py_DECREF(ele);
       } else {
         PyTuple_SET_ITEM(obj->values, keyidx, ele);
         // DECREF(ele) not required as SET_ITEM steals the ref
       }
     }
 
     PyList_SET_ITEM(arrval, i, dict);
     // DECREF(obj) not required as SET_ITEM steals the ref
   }
 
   Py_DECREF(keys);
 
   return arrval;
 }
 
 static PyObject* bser_loads_recursive(
     const char** ptr,
     const char* end,
     const unser_ctx_t* ctx) {
   const char* buf = *ptr;
 
   switch (buf[0]) {
     case BSER_INT8:
     case BSER_INT16:
     case BSER_INT32:
     case BSER_INT64: {
       int64_t ival;
       if (!bunser_int(ptr, end, &ival)) {
         return NULL;
       }
 // Python 3 has one integer type.
 #if PY_MAJOR_VERSION >= 3
       return PyLong_FromLongLong(ival);
 #else
       if (ival < LONG_MIN || ival > LONG_MAX) {
         return PyLong_FromLongLong(ival);
       }
       return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t));
 #endif // PY_MAJOR_VERSION >= 3
     }
 
     case BSER_REAL: {
       double dval;
       memcpy(&dval, buf + 1, sizeof(dval));
       *ptr = buf + 1 + sizeof(double);
       return PyFloat_FromDouble(dval);
     }
 
     case BSER_TRUE:
       *ptr = buf + 1;
       Py_INCREF(Py_True);
       return Py_True;
 
     case BSER_FALSE:
       *ptr = buf + 1;
       Py_INCREF(Py_False);
       return Py_False;
 
     case BSER_NULL:
       *ptr = buf + 1;
       Py_INCREF(Py_None);
       return Py_None;
 
     case BSER_BYTESTRING: {
       const char* start;
       int64_t len;
 
       if (!bunser_bytestring(ptr, end, &start, &len)) {
         return NULL;
       }
 
       if (len > LONG_MAX) {
         PyErr_Format(PyExc_ValueError, "string too long for python");
         return NULL;
       }
 
       if (ctx->value_encoding != NULL) {
         return PyUnicode_Decode(
             start, (long)len, ctx->value_encoding, ctx->value_errors);
       } else {
         return PyBytes_FromStringAndSize(start, (long)len);
       }
     }
 
     case BSER_UTF8STRING: {
       const char* start;
       int64_t len;
 
       if (!bunser_bytestring(ptr, end, &start, &len)) {
         return NULL;
       }
 
       if (len > LONG_MAX) {
         PyErr_Format(PyExc_ValueError, "string too long for python");
         return NULL;
       }
 
       return PyUnicode_Decode(start, (long)len, "utf-8", "strict");
     }
 
     case BSER_ARRAY:
       return bunser_array(ptr, end, ctx);
 
     case BSER_OBJECT:
       return bunser_object(ptr, end, ctx);
 
     case BSER_TEMPLATE:
       return bunser_template(ptr, end, ctx);
 
     default:
       PyErr_Format(PyExc_ValueError, "unhandled bser opcode 0x%02x", buf[0]);
   }
 
   return NULL;
 }
 
 static int _pdu_info_helper(
     const char* data,
     const char* end,
     uint32_t* bser_version_out,
     uint32_t* bser_capabilities_out,
     int64_t* expected_len_out,
     off_t* position_out) {
   uint32_t bser_version;
   uint32_t bser_capabilities = 0;
   int64_t expected_len;
 
   const char* start;
   start = data;
   // Validate the header and length
   if (memcmp(data, EMPTY_HEADER, 2) == 0) {
     bser_version = 1;
   } else if (memcmp(data, EMPTY_HEADER_V2, 2) == 0) {
     bser_version = 2;
   } else {
     PyErr_SetString(PyExc_ValueError, "invalid bser header");
     return 0;
   }
 
   data += 2;
 
   if (bser_version == 2) {
     // Expect an integer telling us what capabilities are supported by the
     // remote server (currently unused).
     if (!memcpy(&bser_capabilities, &data, sizeof(bser_capabilities))) {
       return 0;
     }
     data += sizeof(bser_capabilities);
   }
 
   // Expect an integer telling us how big the rest of the data
   // should be
   if (!bunser_int(&data, end, &expected_len)) {
     return 0;
   }
 
   *bser_version_out = bser_version;
   *bser_capabilities_out = (uint32_t)bser_capabilities;
   *expected_len_out = expected_len;
   *position_out = (off_t)(data - start);
   return 1;
 }
 
 // This function parses the PDU header and provides info about the packet
 // Returns false if unsuccessful
 static int pdu_info_helper(
     PyObject* self,
     PyObject* args,
     uint32_t* bser_version_out,
     uint32_t* bser_capabilities_out,
     int64_t* total_len_out) {
   const char* start = NULL;
   const char* data = NULL;
   int datalen = 0;
   const char* end;
   int64_t expected_len;
   off_t position;
 
   if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) {
     return 0;
   }
   data = start;
   end = data + datalen;
 
   if (!_pdu_info_helper(
           data,
           end,
           bser_version_out,
           bser_capabilities_out,
           &expected_len,
           &position)) {
     return 0;
   }
   *total_len_out = (int64_t)(expected_len + position);
   return 1;
 }
 
 // Expected use case is to read a packet from the socket and then call
 // bser.pdu_info on the packet.  It returns the BSER version, BSER capabilities,
 // and the total length of the entire response that the peer is sending,
 // including the bytes already received. This allows the client  to compute the
 // data size it needs to read before it can decode the data.
 static PyObject* bser_pdu_info(PyObject* self, PyObject* args) {
   uint32_t version, capabilities;
   int64_t total_len;
   if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) {
     return NULL;
   }
   return Py_BuildValue("kkL", version, capabilities, total_len);
 }
 
 static PyObject* bser_pdu_len(PyObject* self, PyObject* args) {
   uint32_t version, capabilities;
   int64_t total_len;
   if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) {
     return NULL;
   }
   return Py_BuildValue("L", total_len);
 }
 
 static PyObject* bser_loads(PyObject* self, PyObject* args, PyObject* kw) {
   const char* data = NULL;
   int datalen = 0;
   const char* start;
   const char* end;
   int64_t expected_len;
   off_t position;
   PyObject* mutable_obj = NULL;
   const char* value_encoding = NULL;
   const char* value_errors = NULL;
   unser_ctx_t ctx = {1, 0};
 
   static char* kw_list[] = {
       "buf", "mutable", "value_encoding", "value_errors", NULL};
 
   if (!PyArg_ParseTupleAndKeywords(
           args,
           kw,
           "s#|Ozz:loads",
           kw_list,
           &start,
           &datalen,
           &mutable_obj,
           &value_encoding,
           &value_errors)) {
     return NULL;
   }
 
   if (mutable_obj) {
     ctx.mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0;
   }
   ctx.value_encoding = value_encoding;
   if (value_encoding == NULL) {
     ctx.value_errors = NULL;
   } else if (value_errors == NULL) {
     ctx.value_errors = "strict";
   } else {
     ctx.value_errors = value_errors;
   }
   data = start;
   end = data + datalen;
 
   if (!_pdu_info_helper(
           data,
           end,
           &ctx.bser_version,
           &ctx.bser_capabilities,
           &expected_len,
           &position)) {
     return NULL;
   }
 
   data = start + position;
   // Verify
   if (expected_len + data != end) {
     PyErr_SetString(PyExc_ValueError, "bser data len != header len");
     return NULL;
   }
 
   return bser_loads_recursive(&data, end, &ctx);
 }
 
 static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) {
-  PyObject *load, *string;
+  PyObject* load;
+  PyObject* load_method;
+  PyObject* string;
+  PyObject* load_method_args;
+  PyObject* load_method_kwargs;
   PyObject* fp = NULL;
   PyObject* mutable_obj = NULL;
-  const char* value_encoding = NULL;
-  const char* value_errors = NULL;
+  PyObject* value_encoding = NULL;
+  PyObject* value_errors = NULL;
 
   static char* kw_list[] = {
       "fp", "mutable", "value_encoding", "value_errors", NULL};
 
   if (!PyArg_ParseTupleAndKeywords(
           args,
           kw,
-          "OOzz:load",
+          "O|OOO:load",
           kw_list,
           &fp,
           &mutable_obj,
           &value_encoding,
           &value_errors)) {
     return NULL;
   }
 
   load = PyImport_ImportModule("pywatchman.load");
   if (load == NULL) {
     return NULL;
   }
-  string = PyObject_CallMethod(
-      load, "load", "OOzz", fp, mutable_obj, value_encoding, value_errors);
+  load_method = PyObject_GetAttrString(load, "load");
+  if (load_method == NULL) {
+    return NULL;
+  }
+  // Mandatory method arguments
+  load_method_args = Py_BuildValue("(O)", fp);
+  if (load_method_args == NULL) {
+    return NULL;
+  }
+  // Optional method arguments
+  load_method_kwargs = PyDict_New();
+  if (load_method_kwargs == NULL) {
+    return NULL;
+  }
+  if (mutable_obj) {
+    PyDict_SetItemString(load_method_kwargs, "mutable", mutable_obj);
+  }
+  if (value_encoding) {
+    PyDict_SetItemString(load_method_kwargs, "value_encoding", value_encoding);
+  }
+  if (value_errors) {
+    PyDict_SetItemString(load_method_kwargs, "value_errors", value_errors);
+  }
+  string = PyObject_Call(load_method, load_method_args, load_method_kwargs);
+  Py_DECREF(load_method_kwargs);
+  Py_DECREF(load_method_args);
+  Py_DECREF(load_method);
   Py_DECREF(load);
   return string;
 }
 
 // clang-format off
 static PyMethodDef bser_methods[] = {
   {"loads", (PyCFunction)bser_loads, METH_VARARGS | METH_KEYWORDS,
    "Deserialize string."},
   {"load", (PyCFunction)bser_load, METH_VARARGS | METH_KEYWORDS,
    "Deserialize a file object"},
   {"pdu_info", (PyCFunction)bser_pdu_info, METH_VARARGS,
    "Extract PDU information."},
   {"pdu_len", (PyCFunction)bser_pdu_len, METH_VARARGS,
    "Extract total PDU length."},
   {"dumps",  (PyCFunction)bser_dumps, METH_VARARGS | METH_KEYWORDS,
    "Serialize string."},
   {NULL, NULL, 0, NULL}
 };
 
 #if PY_MAJOR_VERSION >= 3
 static struct PyModuleDef bser_module = {
   PyModuleDef_HEAD_INIT,
   "bser",
   "Efficient encoding and decoding of BSER.",
   -1,
   bser_methods
 };
 // clang-format on
 
 PyMODINIT_FUNC PyInit_bser(void) {
   PyObject* mod;
 
   mod = PyModule_Create(&bser_module);
   PyType_Ready(&bserObjectType);
 
   return mod;
 }
 #else
 
 PyMODINIT_FUNC initbser(void) {
   (void)Py_InitModule("bser", bser_methods);
   PyType_Ready(&bserObjectType);
 }
 #endif // PY_MAJOR_VERSION >= 3
 
 /* vim:ts=2:sw=2:et:
  */

hgext/fsmonitor/pywatchman/capabilities.py

@@ -1,71 +1,77 b''
 # Copyright 2015 Facebook, Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 #  * Redistributions of source code must retain the above copyright notice,
 #    this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright notice,
 #    this list of conditions and the following disclaimer in the documentation
 #    and/or other materials provided with the distribution.
 #
 #  * Neither the name Facebook nor the names of its contributors may be used to
 #    endorse or promote products derived from this software without specific
 #    prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
 
 import re
 
+
 def parse_version(vstr):
     res = 0
-    for n in vstr.split('.'):
+    for n in vstr.split("."):
         res = res * 1000
         res = res + int(n)
     return res
 
+
 cap_versions = {
     "cmd-watch-del-all": "3.1.1",
     "cmd-watch-project": "3.1",
     "relative_root": "3.3",
     "term-dirname": "3.1",
     "term-idirname": "3.1",
     "wildmatch": "3.7",
 }
 
+
 def check(version, name):
     if name in cap_versions:
         return version >= parse_version(cap_versions[name])
     return False
 
+
 def synthesize(vers, opts):
     """ Synthesize a capability enabled version response
         This is a very limited emulation for relatively recent feature sets
     """
-    parsed_version = parse_version(vers['version'])
-    vers['capabilities'] = {}
-    for name in opts['optional']:
-        vers['capabilities'][name] = check(parsed_version, name)
-    for name in opts['required']:
+    parsed_version = parse_version(vers["version"])
+    vers["capabilities"] = {}
+    for name in opts["optional"]:
+        vers["capabilities"][name] = check(parsed_version, name)
+    failed = False  # noqa: F841 T25377293 Grandfathered in
+    for name in opts["required"]:
         have = check(parsed_version, name)
-        vers['capabilities'][name] = have
+        vers["capabilities"][name] = have
         if not have:
-            vers['error'] = 'client required capability `' + name + \
-                            '` is not supported by this server'
+            vers["error"] = (
+                "client required capability `"
+                + name
+                + "` is not supported by this server"
+            )
     return vers

hgext/fsmonitor/pywatchman/compat.py

@@ -1,65 +1,71 b''
 # Copyright 2016-present Facebook, Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 #  * Redistributions of source code must retain the above copyright notice,
 #    this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright notice,
 #    this list of conditions and the following disclaimer in the documentation
 #    and/or other materials provided with the distribution.
 #
 #  * Neither the name Facebook nor the names of its contributors may be used to
 #    endorse or promote products derived from this software without specific
 #    prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
-
-'''Compatibility module across Python 2 and 3.'''
+from __future__ import absolute_import, division, print_function
 
 import sys
 
+
+"""Compatibility module across Python 2 and 3."""
+
+
+PYTHON2 = sys.version_info < (3, 0)
 PYTHON3 = sys.version_info >= (3, 0)
 
 # This is adapted from https://bitbucket.org/gutworth/six, and used under the
 # MIT license. See LICENSE for a full copyright notice.
 if PYTHON3:
+
     def reraise(tp, value, tb=None):
         try:
             if value is None:
                 value = tp()
             if value.__traceback__ is not tb:
                 raise value.with_traceback(tb)
             raise value
         finally:
             value = None
             tb = None
+
+
 else:
-    exec('''
+    exec(
+        """
 def reraise(tp, value, tb=None):
     try:
         raise tp, value, tb
     finally:
         tb = None
-'''.strip())
+""".strip()
+    )
 
 if PYTHON3:
     UNICODE = str
 else:
-    UNICODE = unicode
+    UNICODE = unicode  # noqa: F821 We handled versioning above

hgext/fsmonitor/pywatchman/encoding.py

@@ -1,73 +1,75 b''
 # Copyright 2016-present Facebook, Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 #  * Redistributions of source code must retain the above copyright notice,
 #    this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright notice,
 #    this list of conditions and the following disclaimer in the documentation
 #    and/or other materials provided with the distribution.
 #
 #  * Neither the name Facebook nor the names of its contributors may be used to
 #    endorse or promote products derived from this software without specific
 #    prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
-
-'''Module to deal with filename encoding on the local system, as returned by
-Watchman.'''
+from __future__ import absolute_import, division, print_function
 
 import sys
 
-from . import (
-    compat,
-)
+from . import compat
+
+
+"""Module to deal with filename encoding on the local system, as returned by
+Watchman."""
+
 
 if compat.PYTHON3:
-    default_local_errors = 'surrogateescape'
+    default_local_errors = "surrogateescape"
 
     def get_local_encoding():
-        if sys.platform == 'win32':
+        if sys.platform == "win32":
             # Watchman always returns UTF-8 encoded strings on Windows.
-            return 'utf-8'
+            return "utf-8"
         # On the Python 3 versions we support, sys.getfilesystemencoding never
         # returns None.
         return sys.getfilesystemencoding()
+
+
 else:
     # Python 2 doesn't support surrogateescape, so use 'strict' by
     # default. Users can register a custom surrogateescape error handler and use
     # that if they so desire.
-    default_local_errors = 'strict'
+    default_local_errors = "strict"
 
     def get_local_encoding():
-        if sys.platform == 'win32':
+        if sys.platform == "win32":
             # Watchman always returns UTF-8 encoded strings on Windows.
-            return 'utf-8'
+            return "utf-8"
         fsencoding = sys.getfilesystemencoding()
         if fsencoding is None:
             # This is very unlikely to happen, but if it does, just use UTF-8
-            fsencoding = 'utf-8'
+            fsencoding = "utf-8"
         return fsencoding
 
+
 def encode_local(s):
     return s.encode(get_local_encoding(), default_local_errors)
 
+
 def decode_local(bs):
     return bs.decode(get_local_encoding(), default_local_errors)

hgext/fsmonitor/pywatchman/load.py

@@ -1,107 +1,109 b''
 # Copyright 2016 Facebook, Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 #  * Redistributions of source code must retain the above copyright notice,
 #    this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright notice,
 #    this list of conditions and the following disclaimer in the documentation
 #    and/or other materials provided with the distribution.
 #
 #  * Neither the name Facebook nor the names of its contributors may be used to
 #    endorse or promote products derived from this software without specific
 #    prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
+
+import ctypes
+
 
 try:
     from . import bser
 except ImportError:
     from . import pybser as bser
 
-import ctypes
 
 EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00"
 
 
 def _read_bytes(fp, buf):
     """Read bytes from a file-like object
 
     @param fp: File-like object that implements read(int)
     @type fp: file
 
     @param buf: Buffer to read into
     @type buf: bytes
 
     @return: buf
     """
 
     # Do the first read without resizing the input buffer
     offset = 0
     remaining = len(buf)
     while remaining > 0:
         l = fp.readinto((ctypes.c_char * remaining).from_buffer(buf, offset))
         if l is None or l == 0:
             return offset
         offset += l
         remaining -= l
     return offset
 
 
 def load(fp, mutable=True, value_encoding=None, value_errors=None):
     """Deserialize a BSER-encoded blob.
 
     @param fp: The file-object to deserialize.
     @type file:
 
     @param mutable: Whether to return mutable results.
     @type mutable: bool
 
     @param value_encoding: Optional codec to use to decode values. If
                            unspecified or None, return values as bytestrings.
     @type value_encoding: str
 
     @param value_errors: Optional error handler for codec. 'strict' by default.
                          The other most common argument is 'surrogateescape' on
                          Python 3. If value_encoding is None, this is ignored.
     @type value_errors: str
     """
     buf = ctypes.create_string_buffer(8192)
     SNIFF_BUFFER_SIZE = len(EMPTY_HEADER)
     header = (ctypes.c_char * SNIFF_BUFFER_SIZE).from_buffer(buf)
     read_len = _read_bytes(fp, header)
     if read_len < len(header):
         return None
 
     total_len = bser.pdu_len(buf)
     if total_len > len(buf):
         ctypes.resize(buf, total_len)
 
     body = (ctypes.c_char * (total_len - len(header))).from_buffer(
-        buf, len(header))
+        buf, len(header)
+    )
     read_len = _read_bytes(fp, body)
     if read_len < len(body):
-        raise RuntimeError('bser data ended early')
+        raise RuntimeError("bser data ended early")
 
     return bser.loads(
         (ctypes.c_char * total_len).from_buffer(buf, 0),
         mutable,
         value_encoding,
-        value_errors)
+        value_errors,
+    )

hgext/fsmonitor/pywatchman/pybser.py

@@ -1,483 +1,576 b''
 # Copyright 2015 Facebook, Inc.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
 #
 #  * Redistributions of source code must retain the above copyright notice,
 #    this list of conditions and the following disclaimer.
 #
 #  * Redistributions in binary form must reproduce the above copyright notice,
 #    this list of conditions and the following disclaimer in the documentation
 #    and/or other materials provided with the distribution.
 #
 #  * Neither the name Facebook nor the names of its contributors may be used to
 #    endorse or promote products derived from this software without specific
 #    prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 # no unicode literals
+from __future__ import absolute_import, division, print_function
 
 import binascii
 import collections
 import ctypes
 import struct
 import sys
 
-from . import (
-    compat,
-)
+from . import compat
+
 
-BSER_ARRAY = b'\x00'
-BSER_OBJECT = b'\x01'
-BSER_BYTESTRING = b'\x02'
-BSER_INT8 = b'\x03'
-BSER_INT16 = b'\x04'
-BSER_INT32 = b'\x05'
-BSER_INT64 = b'\x06'
-BSER_REAL = b'\x07'
-BSER_TRUE = b'\x08'
-BSER_FALSE = b'\x09'
-BSER_NULL = b'\x0a'
-BSER_TEMPLATE = b'\x0b'
-BSER_SKIP = b'\x0c'
-BSER_UTF8STRING = b'\x0d'
+BSER_ARRAY = b"\x00"
+BSER_OBJECT = b"\x01"
+BSER_BYTESTRING = b"\x02"
+BSER_INT8 = b"\x03"
+BSER_INT16 = b"\x04"
+BSER_INT32 = b"\x05"
+BSER_INT64 = b"\x06"
+BSER_REAL = b"\x07"
+BSER_TRUE = b"\x08"
+BSER_FALSE = b"\x09"
+BSER_NULL = b"\x0a"
+BSER_TEMPLATE = b"\x0b"
+BSER_SKIP = b"\x0c"
+BSER_UTF8STRING = b"\x0d"
 
 if compat.PYTHON3:
     STRING_TYPES = (str, bytes)
     unicode = str
+
     def tobytes(i):
-        return str(i).encode('ascii')
+        return str(i).encode("ascii")
+
     long = int
 else:
     STRING_TYPES = (unicode, str)
     tobytes = bytes
 
 # Leave room for the serialization header, which includes
 # our overall length.  To make things simpler, we'll use an
 # int32 for the header
 EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00"
 EMPTY_HEADER_V2 = b"\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00"
 
+
 def _int_size(x):
     """Return the smallest size int that can store the value"""
     if -0x80 <= x <= 0x7F:
         return 1
     elif -0x8000 <= x <= 0x7FFF:
         return 2
     elif -0x80000000 <= x <= 0x7FFFFFFF:
         return 4
     elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF):
         return 8
     else:
-        raise RuntimeError('Cannot represent value: ' + str(x))
+        raise RuntimeError("Cannot represent value: " + str(x))
+
 
 def _buf_pos(buf, pos):
     ret = buf[pos]
-    # In Python 2, buf is a str array so buf[pos] is a string. In Python 3, buf
-    # is a bytes array and buf[pos] is an integer.
-    if compat.PYTHON3:
+    # Normalize the return type to bytes
+    if compat.PYTHON3 and not isinstance(ret, bytes):
         ret = bytes((ret,))
     return ret
 
+
 class _bser_buffer(object):
-
     def __init__(self, version):
         self.bser_version = version
         self.buf = ctypes.create_string_buffer(8192)
         if self.bser_version == 1:
-            struct.pack_into(tobytes(len(EMPTY_HEADER)) + b's', self.buf, 0,
-                             EMPTY_HEADER)
+            struct.pack_into(
+                tobytes(len(EMPTY_HEADER)) + b"s", self.buf, 0, EMPTY_HEADER
+            )
             self.wpos = len(EMPTY_HEADER)
         else:
             assert self.bser_version == 2
-            struct.pack_into(tobytes(len(EMPTY_HEADER_V2)) + b's', self.buf, 0,
-                             EMPTY_HEADER_V2)
+            struct.pack_into(
+                tobytes(len(EMPTY_HEADER_V2)) + b"s",
+                self.buf,
+                0,
+                EMPTY_HEADER_V2,
+            )
             self.wpos = len(EMPTY_HEADER_V2)
 
     def ensure_size(self, size):
         while ctypes.sizeof(self.buf) - self.wpos < size:
             ctypes.resize(self.buf, ctypes.sizeof(self.buf) * 2)
 
     def append_long(self, val):
         size = _int_size(val)
         to_write = size + 1
         self.ensure_size(to_write)
         if size == 1:
-            struct.pack_into(b'=cb', self.buf, self.wpos, BSER_INT8, val)
+            struct.pack_into(b"=cb", self.buf, self.wpos, BSER_INT8, val)
         elif size == 2:
-            struct.pack_into(b'=ch', self.buf, self.wpos, BSER_INT16, val)
+            struct.pack_into(b"=ch", self.buf, self.wpos, BSER_INT16, val)
         elif size == 4:
-            struct.pack_into(b'=ci', self.buf, self.wpos, BSER_INT32, val)
+            struct.pack_into(b"=ci", self.buf, self.wpos, BSER_INT32, val)
         elif size == 8:
-            struct.pack_into(b'=cq', self.buf, self.wpos, BSER_INT64, val)
+            struct.pack_into(b"=cq", self.buf, self.wpos, BSER_INT64, val)
         else:
-            raise RuntimeError('Cannot represent this long value')
+            raise RuntimeError("Cannot represent this long value")
         self.wpos += to_write
 
-
     def append_string(self, s):
         if isinstance(s, unicode):
-            s = s.encode('utf-8')
+            s = s.encode("utf-8")
         s_len = len(s)
         size = _int_size(s_len)
         to_write = 2 + size + s_len
         self.ensure_size(to_write)
         if size == 1:
-            struct.pack_into(b'=ccb' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT8, s_len, s)
+            struct.pack_into(
+                b"=ccb" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT8,
+                s_len,
+                s,
+            )
         elif size == 2:
-            struct.pack_into(b'=cch' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT16, s_len, s)
+            struct.pack_into(
+                b"=cch" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT16,
+                s_len,
+                s,
+            )
         elif size == 4:
-            struct.pack_into(b'=cci' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT32, s_len, s)
+            struct.pack_into(
+                b"=cci" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT32,
+                s_len,
+                s,
+            )
         elif size == 8:
-            struct.pack_into(b'=ccq' + tobytes(s_len) + b's', self.buf,
-                self.wpos, BSER_BYTESTRING, BSER_INT64, s_len, s)
+            struct.pack_into(
+                b"=ccq" + tobytes(s_len) + b"s",
+                self.buf,
+                self.wpos,
+                BSER_BYTESTRING,
+                BSER_INT64,
+                s_len,
+                s,
+            )
         else:
-            raise RuntimeError('Cannot represent this string value')
+            raise RuntimeError("Cannot represent this string value")
         self.wpos += to_write
 
-
     def append_recursive(self, val):
         if isinstance(val, bool):
             needed = 1
             self.ensure_size(needed)
             if val:
                 to_encode = BSER_TRUE
             else:
                 to_encode = BSER_FALSE
-            struct.pack_into(b'=c', self.buf, self.wpos, to_encode)
+            struct.pack_into(b"=c", self.buf, self.wpos, to_encode)
             self.wpos += needed
         elif val is None:
             needed = 1
             self.ensure_size(needed)
-            struct.pack_into(b'=c', self.buf, self.wpos, BSER_NULL)
+            struct.pack_into(b"=c", self.buf, self.wpos, BSER_NULL)
             self.wpos += needed
         elif isinstance(val, (int, long)):
             self.append_long(val)
         elif isinstance(val, STRING_TYPES):
             self.append_string(val)
         elif isinstance(val, float):
             needed = 9
             self.ensure_size(needed)
-            struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val)
+            struct.pack_into(b"=cd", self.buf, self.wpos, BSER_REAL, val)
             self.wpos += needed
-        elif isinstance(val, collections.Mapping) and \
-            isinstance(val, collections.Sized):
+        elif isinstance(val, collections.Mapping) and isinstance(
+            val, collections.Sized
+        ):
             val_len = len(val)
             size = _int_size(val_len)
             needed = 2 + size
             self.ensure_size(needed)
             if size == 1:
-                struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT8, val_len)
+                struct.pack_into(
+                    b"=ccb",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT8,
+                    val_len,
+                )
             elif size == 2:
-                struct.pack_into(b'=cch', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT16, val_len)
+                struct.pack_into(
+                    b"=cch",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT16,
+                    val_len,
+                )
             elif size == 4:
-                struct.pack_into(b'=cci', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT32, val_len)
+                struct.pack_into(
+                    b"=cci",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT32,
+                    val_len,
+                )
             elif size == 8:
-                struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_OBJECT,
-                    BSER_INT64, val_len)
+                struct.pack_into(
+                    b"=ccq",
+                    self.buf,
+                    self.wpos,
+                    BSER_OBJECT,
+                    BSER_INT64,
+                    val_len,
+                )
             else:
-                raise RuntimeError('Cannot represent this mapping value')
+                raise RuntimeError("Cannot represent this mapping value")
             self.wpos += needed
             if compat.PYTHON3:
                 iteritems = val.items()
             else:
-                iteritems = val.iteritems()
+                iteritems = val.iteritems()  # noqa: B301 Checked version above
             for k, v in iteritems:
                 self.append_string(k)
                 self.append_recursive(v)
-        elif isinstance(val, collections.Iterable) and \
-            isinstance(val, collections.Sized):
+        elif isinstance(val, collections.Iterable) and isinstance(
+            val, collections.Sized
+        ):
             val_len = len(val)
             size = _int_size(val_len)
             needed = 2 + size
             self.ensure_size(needed)
             if size == 1:
-                struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT8, val_len)
+                struct.pack_into(
+                    b"=ccb", self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len
+                )
             elif size == 2:
-                struct.pack_into(b'=cch', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT16, val_len)
+                struct.pack_into(
+                    b"=cch",
+                    self.buf,
+                    self.wpos,
+                    BSER_ARRAY,
+                    BSER_INT16,
+                    val_len,
+                )
             elif size == 4:
-                struct.pack_into(b'=cci', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT32, val_len)
+                struct.pack_into(
+                    b"=cci",
+                    self.buf,
+                    self.wpos,
+                    BSER_ARRAY,
+                    BSER_INT32,
+                    val_len,
+                )
             elif size == 8:
-                struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_ARRAY,
-                    BSER_INT64, val_len)
+                struct.pack_into(
+                    b"=ccq",
+                    self.buf,
+                    self.wpos,
+                    BSER_ARRAY,
+                    BSER_INT64,
+                    val_len,
+                )
             else:
-                raise RuntimeError('Cannot represent this sequence value')
+                raise RuntimeError("Cannot represent this sequence value")
             self.wpos += needed
             for v in val:
                 self.append_recursive(v)
         else:
-            raise RuntimeError('Cannot represent unknown value type')
+            raise RuntimeError("Cannot represent unknown value type")
 
 
 def dumps(obj, version=1, capabilities=0):
     bser_buf = _bser_buffer(version=version)
     bser_buf.append_recursive(obj)
     # Now fill in the overall length
     if version == 1:
         obj_len = bser_buf.wpos - len(EMPTY_HEADER)
-        struct.pack_into(b'=i', bser_buf.buf, 3, obj_len)
+        struct.pack_into(b"=i", bser_buf.buf, 3, obj_len)
     else:
         obj_len = bser_buf.wpos - len(EMPTY_HEADER_V2)
-        struct.pack_into(b'=i', bser_buf.buf, 2, capabilities)
-        struct.pack_into(b'=i', bser_buf.buf, 7, obj_len)
-    return bser_buf.buf.raw[:bser_buf.wpos]
+        struct.pack_into(b"=i", bser_buf.buf, 2, capabilities)
+        struct.pack_into(b"=i", bser_buf.buf, 7, obj_len)
+    return bser_buf.buf.raw[: bser_buf.wpos]
+
 
 # This is a quack-alike with the bserObjectType in bser.c
 # It provides by getattr accessors and getitem for both index
 # and name.
 class _BunserDict(object):
-    __slots__ = ('_keys', '_values')
+    __slots__ = ("_keys", "_values")
 
     def __init__(self, keys, values):
         self._keys = keys
         self._values = values
 
     def __getattr__(self, name):
         return self.__getitem__(name)
 
     def __getitem__(self, key):
         if isinstance(key, (int, long)):
             return self._values[key]
-        elif key.startswith('st_'):
+        elif key.startswith("st_"):
             # hack^Wfeature to allow mercurial to use "st_size" to
             # reference "size"
             key = key[3:]
         try:
             return self._values[self._keys.index(key)]
         except ValueError:
-            raise KeyError('_BunserDict has no key %s' % key)
+            raise KeyError("_BunserDict has no key %s" % key)
 
     def __len__(self):
         return len(self._keys)
 
+
 class Bunser(object):
     def __init__(self, mutable=True, value_encoding=None, value_errors=None):
         self.mutable = mutable
         self.value_encoding = value_encoding
 
         if value_encoding is None:
             self.value_errors = None
         elif value_errors is None:
-            self.value_errors = 'strict'
+            self.value_errors = "strict"
         else:
             self.value_errors = value_errors
 
     @staticmethod
     def unser_int(buf, pos):
         try:
             int_type = _buf_pos(buf, pos)
         except IndexError:
-            raise ValueError('Invalid bser int encoding, pos out of range')
+            raise ValueError("Invalid bser int encoding, pos out of range")
         if int_type == BSER_INT8:
             needed = 2
-            fmt = b'=b'
+            fmt = b"=b"
         elif int_type == BSER_INT16:
             needed = 3
-            fmt = b'=h'
+            fmt = b"=h"
         elif int_type == BSER_INT32:
             needed = 5
-            fmt = b'=i'
+            fmt = b"=i"
         elif int_type == BSER_INT64:
             needed = 9
-            fmt = b'=q'
+            fmt = b"=q"
         else:
-            raise ValueError('Invalid bser int encoding 0x%s' %
-                             binascii.hexlify(int_type).decode('ascii'))
+            raise ValueError(
+                "Invalid bser int encoding 0x%s at position %s"
+                % (binascii.hexlify(int_type).decode("ascii"), pos)
+            )
         int_val = struct.unpack_from(fmt, buf, pos + 1)[0]
         return (int_val, pos + needed)
 
     def unser_utf8_string(self, buf, pos):
         str_len, pos = self.unser_int(buf, pos + 1)
-        str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0]
-        return (str_val.decode('utf-8'), pos + str_len)
+        str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0]
+        return (str_val.decode("utf-8"), pos + str_len)
 
     def unser_bytestring(self, buf, pos):
         str_len, pos = self.unser_int(buf, pos + 1)
-        str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0]
+        str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0]
         if self.value_encoding is not None:
             str_val = str_val.decode(self.value_encoding, self.value_errors)
             # str_len stays the same because that's the length in bytes
         return (str_val, pos + str_len)
 
     def unser_array(self, buf, pos):
         arr_len, pos = self.unser_int(buf, pos + 1)
         arr = []
-        for i in range(arr_len):
+        for _ in range(arr_len):
             arr_item, pos = self.loads_recursive(buf, pos)
             arr.append(arr_item)
 
         if not self.mutable:
-          arr = tuple(arr)
+            arr = tuple(arr)
 
         return arr, pos
 
     def unser_object(self, buf, pos):
         obj_len, pos = self.unser_int(buf, pos + 1)
         if self.mutable:
             obj = {}
         else:
             keys = []
             vals = []
 
-        for i in range(obj_len):
+        for _ in range(obj_len):
             key, pos = self.unser_utf8_string(buf, pos)
             val, pos = self.loads_recursive(buf, pos)
             if self.mutable:
                 obj[key] = val
             else:
                 keys.append(key)
                 vals.append(val)
 
         if not self.mutable:
             obj = _BunserDict(keys, vals)
 
         return obj, pos
 
     def unser_template(self, buf, pos):
         val_type = _buf_pos(buf, pos + 1)
         if val_type != BSER_ARRAY:
-            raise RuntimeError('Expect ARRAY to follow TEMPLATE')
+            raise RuntimeError("Expect ARRAY to follow TEMPLATE")
         # force UTF-8 on keys
-        keys_bunser = Bunser(mutable=self.mutable, value_encoding='utf-8')
+        keys_bunser = Bunser(mutable=self.mutable, value_encoding="utf-8")
         keys, pos = keys_bunser.unser_array(buf, pos + 1)
         nitems, pos = self.unser_int(buf, pos)
         arr = []
-        for i in range(nitems):
+        for _ in range(nitems):
             if self.mutable:
                 obj = {}
             else:
                 vals = []
 
             for keyidx in range(len(keys)):
                 if _buf_pos(buf, pos) == BSER_SKIP:
                     pos += 1
                     ele = None
                 else:
                     ele, pos = self.loads_recursive(buf, pos)
 
                 if self.mutable:
                     key = keys[keyidx]
                     obj[key] = ele
                 else:
                     vals.append(ele)
 
             if not self.mutable:
                 obj = _BunserDict(keys, vals)
 
             arr.append(obj)
         return arr, pos
 
     def loads_recursive(self, buf, pos):
         val_type = _buf_pos(buf, pos)
-        if (val_type == BSER_INT8 or val_type == BSER_INT16 or
-            val_type == BSER_INT32 or val_type == BSER_INT64):
+        if (
+            val_type == BSER_INT8
+            or val_type == BSER_INT16
+            or val_type == BSER_INT32
+            or val_type == BSER_INT64
+        ):
             return self.unser_int(buf, pos)
         elif val_type == BSER_REAL:
-            val = struct.unpack_from(b'=d', buf, pos + 1)[0]
+            val = struct.unpack_from(b"=d", buf, pos + 1)[0]
             return (val, pos + 9)
         elif val_type == BSER_TRUE:
             return (True, pos + 1)
         elif val_type == BSER_FALSE:
             return (False, pos + 1)
         elif val_type == BSER_NULL:
             return (None, pos + 1)
         elif val_type == BSER_BYTESTRING:
             return self.unser_bytestring(buf, pos)
         elif val_type == BSER_UTF8STRING:
             return self.unser_utf8_string(buf, pos)
         elif val_type == BSER_ARRAY:
             return self.unser_array(buf, pos)
         elif val_type == BSER_OBJECT:
             return self.unser_object(buf, pos)
         elif val_type == BSER_TEMPLATE:
             return self.unser_template(buf, pos)
         else:
-            raise ValueError('unhandled bser opcode 0x%s' %
-                             binascii.hexlify(val_type).decode('ascii'))
+            raise ValueError(
+                "unhandled bser opcode 0x%s"
+                % binascii.hexlify(val_type).decode("ascii")
+            )
 
 
 def _pdu_info_helper(buf):
+    bser_version = -1
     if buf[0:2] == EMPTY_HEADER[0:2]:
         bser_version = 1
         bser_capabilities = 0
         expected_len, pos2 = Bunser.unser_int(buf, 2)
     elif buf[0:2] == EMPTY_HEADER_V2[0:2]:
         if len(buf) < 8:
-            raise ValueError('Invalid BSER header')
+            raise ValueError("Invalid BSER header")
         bser_version = 2
         bser_capabilities = struct.unpack_from("I", buf, 2)[0]
         expected_len, pos2 = Bunser.unser_int(buf, 6)
     else:
-        raise ValueError('Invalid BSER header')
+        raise ValueError("Invalid BSER header")
 
     return bser_version, bser_capabilities, expected_len, pos2
 
 
 def pdu_info(buf):
     info = _pdu_info_helper(buf)
     return info[0], info[1], info[2] + info[3]
 
 
 def pdu_len(buf):
     info = _pdu_info_helper(buf)
     return info[2] + info[3]
 
 
 def loads(buf, mutable=True, value_encoding=None, value_errors=None):
     """Deserialize a BSER-encoded blob.
 
     @param buf: The buffer to deserialize.
     @type buf: bytes
 
     @param mutable: Whether to return mutable results.
     @type mutable: bool
 
     @param value_encoding: Optional codec to use to decode values. If
                            unspecified or None, return values as bytestrings.
     @type value_encoding: str
 
     @param value_errors: Optional error handler for codec. 'strict' by default.
                          The other most common argument is 'surrogateescape' on
                          Python 3. If value_encoding is None, this is ignored.
     @type value_errors: str
     """
 
     info = _pdu_info_helper(buf)
     expected_len = info[2]
     pos = info[3]
 
     if len(buf) != expected_len + pos:
-        raise ValueError('bser data len != header len')
+        raise ValueError(
+            "bser data len %d != header len %d" % (expected_len + pos, len(buf))
+        )
 
-    bunser = Bunser(mutable=mutable, value_encoding=value_encoding,
-                    value_errors=value_errors)
+    bunser = Bunser(
+        mutable=mutable,
+        value_encoding=value_encoding,
+        value_errors=value_errors,
+    )
 
     return bunser.loads_recursive(buf, pos)[0]
 
 
 def load(fp, mutable=True, value_encoding=None, value_errors=None):
     from . import load
+
     return load.load(fp, mutable, value_encoding, value_errors)

hgext/fsmonitor/watchmanclient.py

@@ -1,118 +1,119 b''
 # watchmanclient.py - Watchman client for the fsmonitor extension
 #
 # Copyright 2013-2016 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import getpass
 
 from mercurial import util
+from mercurial.utils import procutil
 
 from . import pywatchman
 
 
 class Unavailable(Exception):
     def __init__(self, msg, warn=True, invalidate=False):
         self.msg = msg
         self.warn = warn
         if self.msg == b'timed out waiting for response':
             self.warn = False
         self.invalidate = invalidate
 
     def __str__(self):
         if self.warn:
             return b'warning: Watchman unavailable: %s' % self.msg
         else:
             return b'Watchman unavailable: %s' % self.msg
 
 
 class WatchmanNoRoot(Unavailable):
     def __init__(self, root, msg):
         self.root = root
         super(WatchmanNoRoot, self).__init__(msg)
 
 
 class client(object):
     def __init__(self, ui, root, timeout=1.0):
         err = None
         if not self._user:
             err = b"couldn't get user"
             warn = True
         if self._user in ui.configlist(b'fsmonitor', b'blacklistusers'):
             err = b'user %s in blacklist' % self._user
             warn = False
 
         if err:
             raise Unavailable(err, warn)
 
         self._timeout = timeout
         self._watchmanclient = None
         self._root = root
         self._ui = ui
         self._firsttime = True
 
     def settimeout(self, timeout):
         self._timeout = timeout
         if self._watchmanclient is not None:
             self._watchmanclient.setTimeout(timeout)
 
     def getcurrentclock(self):
         result = self.command(b'clock')
         if not util.safehasattr(result, 'clock'):
             raise Unavailable(
                 b'clock result is missing clock value', invalidate=True
             )
         return result.clock
 
     def clearconnection(self):
         self._watchmanclient = None
 
     def available(self):
         return self._watchmanclient is not None or self._firsttime
 
     @util.propertycache
     def _user(self):
         try:
             return getpass.getuser()
         except KeyError:
             # couldn't figure out our user
             return None
 
     def _command(self, *args):
         watchmanargs = (args[0], self._root) + args[1:]
         try:
             if self._watchmanclient is None:
                 self._firsttime = False
                 watchman_exe = self._ui.configpath(
                     b'fsmonitor', b'watchman_exe'
                 )
                 self._watchmanclient = pywatchman.client(
                     timeout=self._timeout,
                     useImmutableBser=True,
-                    watchman_exe=watchman_exe,
+                    binpath=procutil.tonativestr(watchman_exe),
                 )
             return self._watchmanclient.query(*watchmanargs)
         except pywatchman.CommandError as ex:
             if b'unable to resolve root' in ex.msg:
                 raise WatchmanNoRoot(self._root, ex.msg)
             raise Unavailable(ex.msg)
         except pywatchman.WatchmanError as ex:
             raise Unavailable(str(ex))
 
     def command(self, *args):
         try:
             try:
                 return self._command(*args)
             except WatchmanNoRoot:
                 # this 'watch' command can also raise a WatchmanNoRoot if
                 # watchman refuses to accept this root
                 self._command(b'watch')
                 return self._command(*args)
         except Unavailable:
             # this is in an outer scope to catch Unavailable form any of the
             # above _command calls
             self._watchmanclient = None
             raise