manager.py
443 lines
| 14.9 KiB
| text/x-python
|
PythonLexer
|
r12591 | """Base class to manage a running kernel""" | ||
|
r2606 | |||
|
r16349 | # Copyright (c) IPython Development Team. | ||
# Distributed under the terms of the Modified BSD License. | ||||
|
r2699 | |||
|
r9372 | from __future__ import absolute_import | ||
|
r17821 | from contextlib import contextmanager | ||
|
r16349 | import os | ||
|
r12873 | import re | ||
|
r3027 | import signal | ||
|
r2995 | import sys | ||
|
r2614 | import time | ||
|
r16263 | import warnings | ||
|
r17821 | try: | ||
from queue import Empty # Py 3 | ||||
except ImportError: | ||||
from Queue import Empty # Py 2 | ||||
|
r2606 | |||
import zmq | ||||
|
r2611 | |||
|
r10300 | from IPython.utils.importstring import import_item | ||
|
r12591 | from IPython.utils.localinterfaces import is_local_ip, local_ips | ||
|
r16382 | from IPython.utils.path import get_ipython_dir | ||
|
r4958 | from IPython.utils.traitlets import ( | ||
|
r10300 | Any, Instance, Unicode, List, Bool, Type, DottedObjectName | ||
|
r4958 | ) | ||
|
r9353 | from IPython.kernel import ( | ||
|
r9348 | launch_kernel, | ||
|
r16263 | kernelspec, | ||
|
r9348 | ) | ||
|
r10285 | from .connect import ConnectionFileMixin | ||
|
r9376 | from .zmq.session import Session | ||
|
r10284 | from .managerabc import ( | ||
|
r9121 | KernelManagerABC | ||
) | ||||
|
r2699 | |||
|
r16732 | class KernelManager(ConnectionFileMixin): | ||
|
r10285 | """Manages a single kernel in a subprocess on this host. | ||
|
r4872 | |||
|
r10285 | This version starts kernels with Popen. | ||
|
r2631 | """ | ||
|
r10285 | |||
|
r2611 | # The PyZMQ Context to use for communication with the kernel. | ||
|
r4015 | context = Instance(zmq.Context) | ||
def _context_default(self): | ||||
return zmq.Context.instance() | ||||
|
r2611 | |||
|
r10300 | # the class to create with our `client` method | ||
|
r10384 | client_class = DottedObjectName('IPython.kernel.blocking.BlockingKernelClient') | ||
|
r10300 | client_factory = Type() | ||
def _client_class_changed(self, name, old, new): | ||||
self.client_factory = import_item(str(new)) | ||||
|
r2730 | # The kernel process with which the KernelManager is communicating. | ||
|
r9348 | # generally a Popen instance | ||
|
r10283 | kernel = Any() | ||
|
r16263 | |||
|
r16382 | kernel_spec_manager = Instance(kernelspec.KernelSpecManager) | ||
def _kernel_spec_manager_default(self): | ||||
return kernelspec.KernelSpecManager(ipython_dir=self.ipython_dir) | ||||
|
r17371 | kernel_name = Unicode(kernelspec.NATIVE_KERNEL_NAME) | ||
|
r16263 | |||
kernel_spec = Instance(kernelspec.KernelSpec) | ||||
def _kernel_spec_default(self): | ||||
|
r16382 | return self.kernel_spec_manager.get_kernel_spec(self.kernel_name) | ||
|
r16263 | |||
def _kernel_name_changed(self, name, old, new): | ||||
|
r17371 | if new == 'python': | ||
self.kernel_name = kernelspec.NATIVE_KERNEL_NAME | ||||
# This triggered another run of this function, so we can exit now | ||||
return | ||||
|
r16382 | self.kernel_spec = self.kernel_spec_manager.get_kernel_spec(new) | ||
|
r16333 | self.ipython_kernel = new in {'python', 'python2', 'python3'} | ||
|
r10283 | |||
|
r9348 | kernel_cmd = List(Unicode, config=True, | ||
|
r16263 | help="""DEPRECATED: Use kernel_name instead. | ||
The Popen Command to launch the kernel. | ||||
|
r14820 | Override this if you have a custom kernel. | ||
If kernel_cmd is specified in a configuration file, | ||||
IPython does not pass any arguments to the kernel, | ||||
because it cannot make any assumptions about the | ||||
arguments that the kernel understands. In particular, | ||||
this means that the kernel does not receive the | ||||
option --debug if it given on the IPython command line. | ||||
|
r9348 | """ | ||
) | ||||
|
r10282 | |||
|
r9348 | def _kernel_cmd_changed(self, name, old, new): | ||
|
r16263 | warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to " | ||
"start different kernels.") | ||||
|
r9348 | self.ipython_kernel = False | ||
ipython_kernel = Bool(True) | ||||
|
r16382 | |||
ipython_dir = Unicode() | ||||
def _ipython_dir_default(self): | ||||
return get_ipython_dir() | ||||
|
r2730 | |||
|
r10285 | # Protected traits | ||
|
r10296 | _launch_args = Any() | ||
_control_socket = Any() | ||||
|
r9175 | |||
|
r10309 | _restarter = Any() | ||
|
r10282 | autorestart = Bool(False, config=True, | ||
help="""Should we autorestart the kernel if it dies.""" | ||||
) | ||||
|
r9175 | def __del__(self): | ||
|
r10296 | self._close_control_socket() | ||
|
r9154 | self.cleanup_connection_file() | ||
|
r9152 | |||
|
r2758 | #-------------------------------------------------------------------------- | ||
|
r10282 | # Kernel restarter | ||
#-------------------------------------------------------------------------- | ||||
def start_restarter(self): | ||||
pass | ||||
def stop_restarter(self): | ||||
pass | ||||
|
r10313 | def add_restart_callback(self, callback, event='restart'): | ||
"""register a callback to be called when a kernel is restarted""" | ||||
if self._restarter is None: | ||||
return | ||||
self._restarter.add_callback(callback, event) | ||||
def remove_restart_callback(self, callback, event='restart'): | ||||
"""unregister a callback to be called when a kernel is restarted""" | ||||
if self._restarter is None: | ||||
return | ||||
self._restarter.remove_callback(callback, event) | ||||
|
r10282 | #-------------------------------------------------------------------------- | ||
|
r10300 | # create a Client connected to our Kernel | ||
#-------------------------------------------------------------------------- | ||||
def client(self, **kwargs): | ||||
"""Create a client configured to connect to our kernel""" | ||||
if self.client_factory is None: | ||||
self.client_factory = import_item(self.client_class) | ||||
kw = {} | ||||
kw.update(self.get_connection_info()) | ||||
|
r10309 | kw.update(dict( | ||
connection_file=self.connection_file, | ||||
session=self.session, | ||||
|
r11064 | parent=self, | ||
|
r10309 | )) | ||
|
r10300 | |||
# add kwargs last, for manual overrides | ||||
kw.update(kwargs) | ||||
return self.client_factory(**kw) | ||||
#-------------------------------------------------------------------------- | ||||
|
r9151 | # Kernel management | ||
|
r9121 | #-------------------------------------------------------------------------- | ||
|
r10282 | |||
|
r17748 | def format_kernel_cmd(self, extra_arguments=None): | ||
|
r12873 | """replace templated args (e.g. {connection_file})""" | ||
|
r17748 | extra_arguments = extra_arguments or [] | ||
|
r9348 | if self.kernel_cmd: | ||
|
r17748 | cmd = self.kernel_cmd + extra_arguments | ||
|
r16263 | else: | ||
|
r17748 | cmd = self.kernel_spec.argv + extra_arguments | ||
|
r16263 | |||
|
r9348 | ns = dict(connection_file=self.connection_file) | ||
ns.update(self._launch_args) | ||||
|
r12873 | |||
pat = re.compile(r'\{([A-Za-z0-9_]+)\}') | ||||
def from_ns(match): | ||||
"""Get the key out of ns if it's there, otherwise no change.""" | ||||
return ns.get(match.group(1), match.group()) | ||||
return [ pat.sub(from_ns, arg) for arg in cmd ] | ||||
|
r10283 | |||
|
r9349 | def _launch_kernel(self, kernel_cmd, **kw): | ||
"""actually launch the kernel | ||||
|
r10283 | |||
|
r9349 | override in a subclass to launch kernel subprocesses differently | ||
""" | ||||
return launch_kernel(kernel_cmd, **kw) | ||||
|
r10283 | |||
|
r10300 | # Control socket used for polite kernel shutdown | ||
|
r10296 | def _connect_control_socket(self): | ||
if self._control_socket is None: | ||||
self._control_socket = self.connect_control() | ||||
|
r10300 | self._control_socket.linger = 100 | ||
|
r10296 | |||
def _close_control_socket(self): | ||||
if self._control_socket is None: | ||||
return | ||||
self._control_socket.close() | ||||
self._control_socket = None | ||||
|
r2851 | def start_kernel(self, **kw): | ||
|
r9129 | """Starts a kernel on this host in a separate process. | ||
|
r2686 | |||
|
r2699 | If random ports (port=0) are being used, this method must be called | ||
before the channels are created. | ||||
|
r2758 | |||
|
r13587 | Parameters | ||
---------- | ||||
|
r3784 | **kw : optional | ||
|
r9352 | keyword arguments that are passed down to build the kernel_cmd | ||
and launching the kernel (e.g. Popen kwargs). | ||||
|
r2611 | """ | ||
|
r12591 | if self.transport == 'tcp' and not is_local_ip(self.ip): | ||
|
r3144 | raise RuntimeError("Can only launch a kernel on a local interface. " | ||
|
r2667 | "Make sure that the '*_address' attributes are " | ||
|
r3144 | "configured properly. " | ||
|
r12591 | "Currently valid addresses are: %s" % local_ips() | ||
|
r3144 | ) | ||
|
r10283 | |||
|
r4958 | # write connection file / get default ports | ||
self.write_connection_file() | ||||
|
r4872 | |||
|
r9348 | # save kwargs for use in restart | ||
|
r2851 | self._launch_args = kw.copy() | ||
|
r9348 | # build the Popen cmd | ||
|
r17748 | extra_arguments = kw.pop('extra_arguments', []) | ||
kernel_cmd = self.format_kernel_cmd(extra_arguments=extra_arguments) | ||||
|
r16349 | if self.kernel_cmd: | ||
# If kernel_cmd has been set manually, don't refer to a kernel spec | ||||
env = os.environ | ||||
else: | ||||
# Environment variables from kernel spec are added to os.environ | ||||
env = os.environ.copy() | ||||
env.update(self.kernel_spec.env or {}) | ||||
|
r9348 | # launch the kernel subprocess | ||
|
r16349 | self.kernel = self._launch_kernel(kernel_cmd, env=env, | ||
|
r9349 | ipython_kernel=self.ipython_kernel, | ||
**kw) | ||||
|
r10282 | self.start_restarter() | ||
|
r10296 | self._connect_control_socket() | ||
|
r2686 | |||
|
r16510 | def request_shutdown(self, restart=False): | ||
"""Send a shutdown request via control channel | ||||
On Windows, this just kills kernels instead, because the shutdown | ||||
messages don't work. | ||||
""" | ||||
|
r10296 | content = dict(restart=restart) | ||
msg = self.session.msg("shutdown_request", content=content) | ||||
self.session.send(self._control_socket, msg) | ||||
|
r10285 | |||
|
r16523 | def finish_shutdown(self, waittime=1, pollinterval=0.1): | ||
|
r16510 | """Wait for kernel shutdown, then kill process if it doesn't shutdown. | ||
This does not send shutdown requests - use :meth:`request_shutdown` | ||||
first. | ||||
""" | ||||
|
r16523 | for i in range(int(waittime/pollinterval)): | ||
|
r16510 | if self.is_alive(): | ||
|
r16523 | time.sleep(pollinterval) | ||
|
r16510 | else: | ||
break | ||||
else: | ||||
# OK, we've waited long enough. | ||||
if self.has_kernel: | ||||
self._kill_kernel() | ||||
|
r16515 | def cleanup(self, connection_file=True): | ||
|
r16510 | """Clean up resources when the kernel is shut down""" | ||
|
r16515 | if connection_file: | ||
|
r16510 | self.cleanup_connection_file() | ||
|
r16515 | self.cleanup_ipc_files() | ||
|
r16510 | self._close_control_socket() | ||
|
r9119 | def shutdown_kernel(self, now=False, restart=False): | ||
|
r10283 | """Attempts to the stop the kernel process cleanly. | ||
|
r9129 | |||
This attempts to shutdown the kernels cleanly by: | ||||
1. Sending it a shutdown message over the shell channel. | ||||
2. If that fails, the kernel is shutdown forcibly by sending it | ||||
a signal. | ||||
|
r8487 | |||
|
r13587 | Parameters | ||
---------- | ||||
|
r9129 | now : bool | ||
Should the kernel be forcible killed *now*. This skips the | ||||
first, nice shutdown attempt. | ||||
restart: bool | ||||
Will this kernel be restarted after it is shutdown. When this | ||||
is True, connection files will not be cleaned up. | ||||
|
r2961 | """ | ||
|
r10282 | # Stop monitoring for restarting while we shutdown. | ||
self.stop_restarter() | ||||
|
r16510 | if now: | ||
self._kill_kernel() | ||||
|
r9119 | else: | ||
|
r16510 | self.request_shutdown(restart=restart) | ||
|
r9119 | # Don't send any additional kernel kill messages immediately, to give | ||
# the kernel a chance to properly execute shutdown actions. Wait for at | ||||
# most 1s, checking every 0.1s. | ||||
|
r16523 | self.finish_shutdown() | ||
|
r4872 | |||
|
r16515 | self.cleanup(connection_file=not restart) | ||
|
r4958 | |||
|
r3784 | def restart_kernel(self, now=False, **kw): | ||
"""Restarts a kernel with the arguments that were used to launch it. | ||||
|
r4872 | |||
|
r3784 | If the old kernel was launched with random ports, the same ports will be | ||
|
r9129 | used for the new kernel. The same connection file is used again. | ||
|
r2972 | |||
Parameters | ||||
---------- | ||||
|
r3030 | now : bool, optional | ||
|
r3784 | If True, the kernel is forcefully restarted *immediately*, without | ||
having a chance to do any cleanup action. Otherwise the kernel is | ||||
given 1s to clean up before a forceful restart is issued. | ||||
|
r2972 | |||
|
r3784 | In all cases the kernel is restarted, the only difference is whether | ||
it is given a chance to perform a clean shutdown or not. | ||||
**kw : optional | ||||
|
r9129 | Any options specified here will overwrite those used to launch the | ||
|
r3784 | kernel. | ||
|
r2851 | """ | ||
if self._launch_args is None: | ||||
raise RuntimeError("Cannot restart the kernel. " | ||||
"No previous call to 'start_kernel'.") | ||||
else: | ||||
|
r3784 | # Stop currently running kernel. | ||
|
r9119 | self.shutdown_kernel(now=now, restart=True) | ||
|
r3784 | |||
# Start new kernel. | ||||
self._launch_args.update(kw) | ||||
|
r2915 | self.start_kernel(**self._launch_args) | ||
|
r2851 | |||
|
r2686 | @property | ||
def has_kernel(self): | ||||
|
r9129 | """Has a kernel been started that we are managing.""" | ||
|
r2730 | return self.kernel is not None | ||
|
r2686 | |||
|
r9130 | def _kill_kernel(self): | ||
|
r9129 | """Kill the running kernel. | ||
|
r8487 | |||
|
r9129 | This is a private method, callers should use shutdown_kernel(now=True). | ||
|
r8487 | """ | ||
|
r3026 | if self.has_kernel: | ||
|
r3032 | |||
|
r8487 | # Signal the kernel to terminate (sends SIGKILL on Unix and calls | ||
# TerminateProcess() on Win32). | ||||
|
r3034 | try: | ||
self.kernel.kill() | ||||
|
r7787 | except OSError as e: | ||
|
r3034 | # In Windows, we will get an Access Denied error if the process | ||
# has already terminated. Ignore it. | ||||
|
r3827 | if sys.platform == 'win32': | ||
if e.winerror != 5: | ||||
raise | ||||
# On Unix, we may get an ESRCH error if the process has already | ||||
# terminated. Ignore it. | ||||
else: | ||||
from errno import ESRCH | ||||
if e.errno != ESRCH: | ||||
raise | ||||
|
r8487 | |||
# Block until the kernel terminates. | ||||
self.kernel.wait() | ||||
|
r2730 | self.kernel = None | ||
|
r2639 | else: | ||
|
r2686 | raise RuntimeError("Cannot kill kernel. No kernel is running!") | ||
|
r2611 | |||
|
r3027 | def interrupt_kernel(self): | ||
|
r9129 | """Interrupts the kernel by sending it a signal. | ||
|
r8487 | |||
Unlike ``signal_kernel``, this operation is well supported on all | ||||
platforms. | ||||
|
r3027 | """ | ||
if self.has_kernel: | ||||
if sys.platform == 'win32': | ||||
|
r9487 | from .zmq.parentpoller import ParentPollerWindows as Poller | ||
|
r3027 | Poller.send_interrupt(self.kernel.win32_interrupt_event) | ||
else: | ||||
self.kernel.send_signal(signal.SIGINT) | ||||
else: | ||||
raise RuntimeError("Cannot interrupt kernel. No kernel is running!") | ||||
|
r2611 | def signal_kernel(self, signum): | ||
|
r9129 | """Sends a signal to the kernel. | ||
|
r8487 | |||
Note that since only SIGTERM is supported on Windows, this function is | ||||
only useful on Unix systems. | ||||
|
r3027 | """ | ||
|
r3026 | if self.has_kernel: | ||
|
r2730 | self.kernel.send_signal(signum) | ||
|
r2686 | else: | ||
raise RuntimeError("Cannot signal kernel. No kernel is running!") | ||||
|
r2611 | |||
|
r2699 | def is_alive(self): | ||
"""Is the kernel process still running?""" | ||||
|
r3026 | if self.has_kernel: | ||
|
r2730 | if self.kernel.poll() is None: | ||
|
r2699 | return True | ||
else: | ||||
return False | ||||
else: | ||||
|
r10285 | # we don't have a kernel | ||
return False | ||||
|
r2699 | |||
|
r9121 | KernelManagerABC.register(KernelManager) | ||
|
r10285 | |||
|
r17821 | |||
def start_new_kernel(startup_timeout=60, kernel_name='python', **kwargs): | ||||
"""Start a new kernel, and return its Manager and Client""" | ||||
km = KernelManager(kernel_name=kernel_name) | ||||
km.start_kernel(**kwargs) | ||||
kc = km.client() | ||||
kc.start_channels() | ||||
|
r19216 | kc.wait_for_ready() | ||
|
r17821 | |||
return km, kc | ||||
@contextmanager | ||||
def run_kernel(**kwargs): | ||||
"""Context manager to create a kernel in a subprocess. | ||||
The kernel is shut down when the context exits. | ||||
Returns | ||||
------- | ||||
kernel_client: connected KernelClient instance | ||||
""" | ||||
km, kc = start_new_kernel(**kwargs) | ||||
try: | ||||
yield kc | ||||
finally: | ||||
kc.stop_channels() | ||||
km.shutdown_kernel(now=True) | ||||