##// END OF EJS Templates
procutil: avoid using os.fork() to implement runbgcommand...
procutil: avoid using os.fork() to implement runbgcommand We ran into the following deadlock: - some command creates an ssh peer, then raises without explicitly closing the peer (hg id + extension in our case) - dispatch catches the exception, calls ui.log('commandfinish', ..) (the sshpeer is still not closed), which calls logtoprocess, which calls procutil.runbgcommand. - in the child of runbgcommand's fork(), between the fork and the exec, the opening of file descriptors triggers a gc which runs the destructor for sshpeer, which waits on ssh's stderr being closed, which never happens since ssh's stderr is held open by the parent of the fork where said destructor hasn't run Remotefilelog appears to have a hack around this deadlock as well. I don't know if there's more subtlety to it, because even though the problem is determistic, it is very fragile, so I didn't manage to reduce it. I can imagine three ways of tackling this problem: 1. don't run any python between fork and exec in runbgcommand 2. make the finalizer harmless after the fork 3. close the peer without relying on gc behavior This commit goes with 1, as forking without exec'ing is tricky in general in a language with gc finalizers. And maybe it's better in the presence of rust threads. A future commit will try 2 or 3. Performance wise: at low memory usage, it's an improvement. At higher memory usage, it's about 2x faster than before when ensurestart=True, but 2x slower when ensurestart=False. Not sure if that matters. The reason for that last bit is that the subprocess.Popen always waits for the execve to finish, and at high memory usage, execve is slow because it deallocates the large page table. Numbers and script: before after mem=1.0GB, ensurestart=True 52.1ms 26.0ms mem=1.0GB, ensurestart=False 14.7ms 26.0ms mem=0.5GB, ensurestart=True 23.2ms 11.2ms mem=0.5GB, ensurestart=False 6.2ms 11.3ms mem=0.2GB, ensurestart=True 15.7ms 7.4ms mem=0.2GB, ensurestart=False 4.3ms 8.1ms mem=0.0GB, ensurestart=True 2.3ms 0.7ms mem=0.0GB, ensurestart=False 0.8ms 0.8ms import time for memsize in [1_000_000_000, 500_000_000, 250_000_000, 0]: mem = 'a' * memsize for ensurestart in [True, False]: now = time.time() n = 100 for i in range(n): procutil.runbgcommand([b'true'], {}, ensurestart=ensurestart) after = time.time() ms = (after - now) / float(n) * 1000 print(f'mem={memsize / 1e9:.1f}GB, ensurestart={ensurestart} -> {ms:.1f}ms') Differential Revision: https://phab.mercurial-scm.org/D9019

File last commit:

r47106:ab2d02e1 default
r47651:8759e22f default
Show More
pythoncapi_compat.h
283 lines | 6.5 KiB | text/x-c | CLexer
// Header file providing new functions of the Python C API to old Python
// versions.
//
// File distributed under the MIT license.
//
// Homepage:
// https://github.com/pythoncapi/pythoncapi_compat
//
// Latest version:
// https://raw.githubusercontent.com/pythoncapi/pythoncapi_compat/master/pythoncapi_compat.h
#ifndef PYTHONCAPI_COMPAT
#define PYTHONCAPI_COMPAT
#ifdef __cplusplus
extern "C" {
#endif
#include <Python.h>
#include "frameobject.h" // PyFrameObject, PyFrame_GetBack()
/* VC 2008 doesn't know about the inline keyword. */
#if defined(_MSC_VER) && _MSC_VER < 1900
#define inline __forceinline
#endif
// Cast argument to PyObject* type.
#ifndef _PyObject_CAST
# define _PyObject_CAST(op) ((PyObject*)(op))
#endif
// bpo-42262 added Py_NewRef() to Python 3.10.0a3
#if PY_VERSION_HEX < 0x030a00A3 && !defined(Py_NewRef)
static inline PyObject* _Py_NewRef(PyObject *obj)
{
Py_INCREF(obj);
return obj;
}
#define Py_NewRef(obj) _Py_NewRef(_PyObject_CAST(obj))
#endif
// bpo-42262 added Py_XNewRef() to Python 3.10.0a3
#if PY_VERSION_HEX < 0x030a00A3 && !defined(Py_XNewRef)
static inline PyObject* _Py_XNewRef(PyObject *obj)
{
Py_XINCREF(obj);
return obj;
}
#define Py_XNewRef(obj) _Py_XNewRef(_PyObject_CAST(obj))
#endif
// bpo-39573 added Py_SET_REFCNT() to Python 3.9.0a4
#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_REFCNT)
static inline void _Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt)
{
ob->ob_refcnt = refcnt;
}
#define Py_SET_REFCNT(ob, refcnt) _Py_SET_REFCNT((PyObject*)(ob), refcnt)
#endif
// bpo-39573 added Py_SET_TYPE() to Python 3.9.0a4
#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_TYPE)
static inline void
_Py_SET_TYPE(PyObject *ob, PyTypeObject *type)
{
ob->ob_type = type;
}
#define Py_SET_TYPE(ob, type) _Py_SET_TYPE((PyObject*)(ob), type)
#endif
// bpo-39573 added Py_SET_SIZE() to Python 3.9.0a4
#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_SET_SIZE)
static inline void
_Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size)
{
ob->ob_size = size;
}
#define Py_SET_SIZE(ob, size) _Py_SET_SIZE((PyVarObject*)(ob), size)
#endif
// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1
#if PY_VERSION_HEX < 0x030900B1
static inline PyCodeObject*
PyFrame_GetCode(PyFrameObject *frame)
{
PyCodeObject *code;
assert(frame != NULL);
code = frame->f_code;
assert(code != NULL);
Py_INCREF(code);
return code;
}
#endif
static inline PyCodeObject*
_PyFrame_GetCodeBorrow(PyFrameObject *frame)
{
PyCodeObject *code = PyFrame_GetCode(frame);
Py_DECREF(code);
return code; // borrowed reference
}
// bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1
#if PY_VERSION_HEX < 0x030900B1
static inline PyFrameObject*
PyFrame_GetBack(PyFrameObject *frame)
{
PyFrameObject *back;
assert(frame != NULL);
back = frame->f_back;
Py_XINCREF(back);
return back;
}
#endif
static inline PyFrameObject*
_PyFrame_GetBackBorrow(PyFrameObject *frame)
{
PyFrameObject *back = PyFrame_GetBack(frame);
Py_XDECREF(back);
return back; // borrowed reference
}
// bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5
#if PY_VERSION_HEX < 0x030900A5
static inline PyInterpreterState *
PyThreadState_GetInterpreter(PyThreadState *tstate)
{
assert(tstate != NULL);
return tstate->interp;
}
#endif
// bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1
#if PY_VERSION_HEX < 0x030900B1
static inline PyFrameObject*
PyThreadState_GetFrame(PyThreadState *tstate)
{
PyFrameObject *frame;
assert(tstate != NULL);
frame = tstate->frame;
Py_XINCREF(frame);
return frame;
}
#endif
static inline PyFrameObject*
_PyThreadState_GetFrameBorrow(PyThreadState *tstate)
{
PyFrameObject *frame = PyThreadState_GetFrame(tstate);
Py_XDECREF(frame);
return frame; // borrowed reference
}
// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5
#if PY_VERSION_HEX < 0x030900A5
static inline PyInterpreterState *
PyInterpreterState_Get(void)
{
PyThreadState *tstate;
PyInterpreterState *interp;
tstate = PyThreadState_GET();
if (tstate == NULL) {
Py_FatalError("GIL released (tstate is NULL)");
}
interp = tstate->interp;
if (interp == NULL) {
Py_FatalError("no current interpreter");
}
return interp;
}
#endif
// bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a6
#if 0x030700A1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030900A6
static inline uint64_t
PyThreadState_GetID(PyThreadState *tstate)
{
assert(tstate != NULL);
return tstate->id;
}
#endif
// bpo-37194 added PyObject_CallNoArgs() to Python 3.9.0a1
#if PY_VERSION_HEX < 0x030900A1
static inline PyObject*
PyObject_CallNoArgs(PyObject *func)
{
return PyObject_CallFunctionObjArgs(func, NULL);
}
#endif
// bpo-39245 made PyObject_CallOneArg() public (previously called
// _PyObject_CallOneArg) in Python 3.9.0a4
#if PY_VERSION_HEX < 0x030900A4
static inline PyObject*
PyObject_CallOneArg(PyObject *func, PyObject *arg)
{
return PyObject_CallFunctionObjArgs(func, arg, NULL);
}
#endif
// bpo-40024 added PyModule_AddType() to Python 3.9.0a5
#if PY_VERSION_HEX < 0x030900A5
static inline int
PyModule_AddType(PyObject *module, PyTypeObject *type)
{
const char *name, *dot;
if (PyType_Ready(type) < 0) {
return -1;
}
// inline _PyType_Name()
name = type->tp_name;
assert(name != NULL);
dot = strrchr(name, '.');
if (dot != NULL) {
name = dot + 1;
}
Py_INCREF(type);
if (PyModule_AddObject(module, name, (PyObject *)type) < 0) {
Py_DECREF(type);
return -1;
}
return 0;
}
#endif
// bpo-40241 added PyObject_GC_IsTracked() to Python 3.9.0a6.
// bpo-4688 added _PyObject_GC_IS_TRACKED() to Python 2.7.0a2.
#if PY_VERSION_HEX < 0x030900A6
static inline int
PyObject_GC_IsTracked(PyObject* obj)
{
return (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj));
}
#endif
// bpo-40241 added PyObject_GC_IsFinalized() to Python 3.9.0a6.
// bpo-18112 added _PyGCHead_FINALIZED() to Python 3.4.0 final.
#if PY_VERSION_HEX < 0x030900A6 && PY_VERSION_HEX >= 0x030400F0
static inline int
PyObject_GC_IsFinalized(PyObject *obj)
{
return (PyObject_IS_GC(obj) && _PyGCHead_FINALIZED((PyGC_Head *)(obj)-1));
}
#endif
// bpo-39573 added Py_IS_TYPE() to Python 3.9.0a4
#if PY_VERSION_HEX < 0x030900A4 && !defined(Py_IS_TYPE)
static inline int
_Py_IS_TYPE(const PyObject *ob, const PyTypeObject *type) {
return ob->ob_type == type;
}
#define Py_IS_TYPE(ob, type) _Py_IS_TYPE((const PyObject*)(ob), type)
#endif
#ifdef __cplusplus
}
#endif
#endif // PYTHONCAPI_COMPAT