From a1c5e3710712a66c9c85f6acee026dd23d9e1c37 2012-06-20 20:35:56 From: Min RK Date: 2012-06-20 20:35:56 Subject: [PATCH] Merge pull request #1981 from tkf/kill-bg-processes Clean BG processes created by %%script on kernel exit * uses less forceful shutdown of kernels in the notebook, allowing atexit machinery to fire * enables daemon BackgroundJobs * cleanup %%script --bg subprocesses at shutdown --- diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py index 4660941..15dc30a 100644 --- a/IPython/core/magics/script.py +++ b/IPython/core/magics/script.py @@ -18,6 +18,7 @@ import sys import signal import time from subprocess import Popen, PIPE +import atexit # Our own packages from IPython.config.configurable import Configurable @@ -134,6 +135,11 @@ class ScriptMagics(Magics, Configurable): self._generate_script_magics() Magics.__init__(self, shell=shell) self.job_manager = BackgroundJobManager() + self.bg_processes = [] + atexit.register(self.kill_bg_processes) + + def __del__(self): + self.kill_bg_processes() def _generate_script_magics(self): cell_magics = self.magics['cell'] @@ -196,11 +202,13 @@ class ScriptMagics(Magics, Configurable): cell = cell.encode('utf8', 'replace') if args.bg: + self.bg_processes.append(p) + self._gc_bg_processes() if args.out: self.shell.user_ns[args.out] = p.stdout if args.err: self.shell.user_ns[args.err] = p.stderr - self.job_manager.new(self._run_script, p, cell) + self.job_manager.new(self._run_script, p, cell, daemon=True) if args.proc: self.shell.user_ns[args.proc] = p return @@ -245,3 +253,36 @@ class ScriptMagics(Magics, Configurable): p.stdin.write(cell) p.stdin.close() p.wait() + + @line_magic("killbgscripts") + def killbgscripts(self, _nouse_=''): + """Kill all BG processes started by %%script and its family.""" + self.kill_bg_processes() + print "All background processes were killed." + + def kill_bg_processes(self): + """Kill all BG processes which are still running.""" + for p in self.bg_processes: + if p.poll() is None: + try: + p.send_signal(signal.SIGINT) + except: + pass + time.sleep(0.1) + for p in self.bg_processes: + if p.poll() is None: + try: + p.terminate() + except: + pass + time.sleep(0.1) + for p in self.bg_processes: + if p.poll() is None: + try: + p.kill() + except: + pass + self._gc_bg_processes() + + def _gc_bg_processes(self): + self.bg_processes = [p for p in self.bg_processes if p.poll() is None] diff --git a/IPython/frontend/html/notebook/handlers.py b/IPython/frontend/html/notebook/handlers.py index 5207aa9..06bfa88 100644 --- a/IPython/frontend/html/notebook/handlers.py +++ b/IPython/frontend/html/notebook/handlers.py @@ -345,7 +345,7 @@ class KernelHandler(AuthenticatedHandler): @web.authenticated def delete(self, kernel_id): km = self.application.kernel_manager - km.kill_kernel(kernel_id) + km.shutdown_kernel(kernel_id) self.set_status(204) self.finish() diff --git a/IPython/frontend/html/notebook/kernelmanager.py b/IPython/frontend/html/notebook/kernelmanager.py index c3a551c..28f6c4d 100644 --- a/IPython/frontend/html/notebook/kernelmanager.py +++ b/IPython/frontend/html/notebook/kernelmanager.py @@ -43,7 +43,7 @@ class MultiKernelManager(LoggingConfigurable): """A class for managing multiple kernels.""" kernel_manager_class = DottedObjectName( - "IPython.zmq.kernelmanager.KernelManager", config=True, + "IPython.zmq.blockingkernelmanager.BlockingKernelManager", config=True, help="""The kernel manager class. This is configurable to allow subclassing of the KernelManager for customized behavior. """ @@ -87,9 +87,22 @@ class MultiKernelManager(LoggingConfigurable): config=self.config, ) km.start_kernel(**kwargs) + # start just the shell channel, needed for graceful restart + km.start_channels(shell=True, sub=False, stdin=False, hb=False) self._kernels[kernel_id] = km return kernel_id + def shutdown_kernel(self, kernel_id): + """Shutdown a kernel by its kernel uuid. + + Parameters + ========== + kernel_id : uuid + The id of the kernel to shutdown. + """ + self.get_kernel(kernel_id).shutdown_kernel() + del self._kernels[kernel_id] + def kill_kernel(self, kernel_id): """Kill a kernel by its kernel uuid. @@ -266,6 +279,13 @@ class MappingKernelManager(MultiKernelManager): self.log.info("Using existing kernel: %s" % kernel_id) return kernel_id + def shutdown_kernel(self, kernel_id): + """Shutdown a kernel and remove its notebook association.""" + self._check_kernel_id(kernel_id) + super(MappingKernelManager, self).shutdown_kernel(kernel_id) + self.delete_mapping_for_kernel(kernel_id) + self.log.info("Kernel shutdown: %s" % kernel_id) + def kill_kernel(self, kernel_id): """Kill a kernel and remove its notebook association.""" self._check_kernel_id(kernel_id) @@ -283,7 +303,7 @@ class MappingKernelManager(MultiKernelManager): """Restart a kernel while keeping clients connected.""" self._check_kernel_id(kernel_id) km = self.get_kernel(kernel_id) - km.restart_kernel(now=True) + km.restart_kernel() self.log.info("Kernel restarted: %s" % kernel_id) return kernel_id diff --git a/IPython/frontend/html/notebook/notebookapp.py b/IPython/frontend/html/notebook/notebookapp.py index a256140..6f588ea 100644 --- a/IPython/frontend/html/notebook/notebookapp.py +++ b/IPython/frontend/html/notebook/notebookapp.py @@ -537,9 +537,9 @@ class NotebookApp(BaseIPythonApplication): """ self.log.info('Shutting down kernels') km = self.kernel_manager - # copy list, since kill_kernel deletes keys + # copy list, since shutdown_kernel deletes keys for kid in list(km.kernel_ids): - km.kill_kernel(kid) + km.shutdown_kernel(kid) def start(self): ip = self.ip if self.ip else '[all ip addresses on your system]' diff --git a/IPython/lib/backgroundjobs.py b/IPython/lib/backgroundjobs.py index 395edd3..8b57bea 100644 --- a/IPython/lib/backgroundjobs.py +++ b/IPython/lib/backgroundjobs.py @@ -140,6 +140,8 @@ class BackgroundJobManager(object): In both cases, the result is stored in the job.result field of the background job object. + You can set `daemon` attribute of the thread by giving the keyword + argument `daemon`. Notes and caveats: @@ -181,7 +183,9 @@ class BackgroundJobManager(object): job = BackgroundJobExpr(func_or_exp, glob, loc) else: raise TypeError('invalid args for new job') - + + if kwargs.get('daemon', False): + job.daemon = True job.num = len(self.all)+1 if self.all else 0 self.running.append(job) self.all[job.num] = job