# HG changeset patch # User Valentin Gatien-Baron # Date 2020-09-14 02:14:25 # Node ID 8759e22f1649cd71a59d05e8f5ef8dce77ca4b44 # Parent 218a26df7813fee76632a483860f54cd09798a13 procutil: avoid using os.fork() to implement runbgcommand We ran into the following deadlock: - some command creates an ssh peer, then raises without explicitly closing the peer (hg id + extension in our case) - dispatch catches the exception, calls ui.log('commandfinish', ..) (the sshpeer is still not closed), which calls logtoprocess, which calls procutil.runbgcommand. - in the child of runbgcommand's fork(), between the fork and the exec, the opening of file descriptors triggers a gc which runs the destructor for sshpeer, which waits on ssh's stderr being closed, which never happens since ssh's stderr is held open by the parent of the fork where said destructor hasn't run Remotefilelog appears to have a hack around this deadlock as well. I don't know if there's more subtlety to it, because even though the problem is determistic, it is very fragile, so I didn't manage to reduce it. I can imagine three ways of tackling this problem: 1. don't run any python between fork and exec in runbgcommand 2. make the finalizer harmless after the fork 3. close the peer without relying on gc behavior This commit goes with 1, as forking without exec'ing is tricky in general in a language with gc finalizers. And maybe it's better in the presence of rust threads. A future commit will try 2 or 3. Performance wise: at low memory usage, it's an improvement. At higher memory usage, it's about 2x faster than before when ensurestart=True, but 2x slower when ensurestart=False. Not sure if that matters. The reason for that last bit is that the subprocess.Popen always waits for the execve to finish, and at high memory usage, execve is slow because it deallocates the large page table. Numbers and script: before after mem=1.0GB, ensurestart=True 52.1ms 26.0ms mem=1.0GB, ensurestart=False 14.7ms 26.0ms mem=0.5GB, ensurestart=True 23.2ms 11.2ms mem=0.5GB, ensurestart=False 6.2ms 11.3ms mem=0.2GB, ensurestart=True 15.7ms 7.4ms mem=0.2GB, ensurestart=False 4.3ms 8.1ms mem=0.0GB, ensurestart=True 2.3ms 0.7ms mem=0.0GB, ensurestart=False 0.8ms 0.8ms import time for memsize in [1_000_000_000, 500_000_000, 250_000_000, 0]: mem = 'a' * memsize for ensurestart in [True, False]: now = time.time() n = 100 for i in range(n): procutil.runbgcommand([b'true'], {}, ensurestart=ensurestart) after = time.time() ms = (after - now) / float(n) * 1000 print(f'mem={memsize / 1e9:.1f}GB, ensurestart={ensurestart} -> {ms:.1f}ms') Differential Revision: https://phab.mercurial-scm.org/D9019 diff --git a/mercurial/utils/procutil.py b/mercurial/utils/procutil.py --- a/mercurial/utils/procutil.py +++ b/mercurial/utils/procutil.py @@ -701,7 +701,88 @@ if pycompat.iswindows: else: - def runbgcommand( + def runbgcommandpy3( + cmd, + env, + shell=False, + stdout=None, + stderr=None, + ensurestart=True, + record_wait=None, + stdin_bytes=None, + ): + """Spawn a command without waiting for it to finish. + + + When `record_wait` is not None, the spawned process will not be fully + detached and the `record_wait` argument will be called with a the + `Subprocess.wait` function for the spawned process. This is mostly + useful for developers that need to make sure the spawned process + finished before a certain point. (eg: writing test)""" + if pycompat.isdarwin: + # avoid crash in CoreFoundation in case another thread + # calls gui() while we're calling fork(). + gui() + + if shell: + script = cmd + else: + if isinstance(cmd, bytes): + cmd = [cmd] + script = b' '.join(shellquote(x) for x in cmd) + if record_wait is None: + # double-fork to completely detach from the parent process + script = b'( %s ) &' % script + start_new_session = True + else: + start_new_session = False + ensurestart = True + + try: + if stdin_bytes is None: + stdin = subprocess.DEVNULL + else: + stdin = pycompat.unnamedtempfile() + stdin.write(stdin_bytes) + stdin.flush() + stdin.seek(0) + if stdout is None: + stdout = subprocess.DEVNULL + if stderr is None: + stderr = subprocess.DEVNULL + + p = subprocess.Popen( + script, + shell=True, + env=env, + close_fds=True, + stdin=stdin, + stdout=stdout, + stderr=stderr, + start_new_session=start_new_session, + ) + except Exception: + if record_wait is not None: + record_wait(255) + raise + finally: + if stdin_bytes is not None: + stdin.close() + if not ensurestart: + # Even though we're not waiting on the child process, + # we still must call waitpid() on it at some point so + # it's not a zombie/defunct. This is especially relevant for + # chg since the parent process won't die anytime soon. + # We use a thread to make the overhead tiny. + t = threading.Thread(target=lambda: p.wait) + t.daemon = True + t.start() + else: + returncode = p.wait + if record_wait is not None: + record_wait(returncode) + + def runbgcommandpy2( cmd, env, shell=False, @@ -811,3 +892,14 @@ else: stdin.close() if record_wait is None: os._exit(returncode) + + if pycompat.ispy3: + # This branch is more robust, because it avoids running python + # code (hence gc finalizers, like sshpeer.__del__, which + # blocks). But we can't easily do the equivalent in py2, + # because of the lack of start_new_session=True flag. Given + # that the py2 branch should die soon, the short-lived + # duplication seems acceptable. + runbgcommand = runbgcommandpy3 + else: + runbgcommand = runbgcommandpy2