##// END OF EJS Templates
Start refactoring KernelManager to use kernel registry
Thomas Kluyver -
Show More
@@ -1,105 +1,107 b''
1 1 import io
2 2 import json
3 3 import os
4 4 import sys
5 5
6 6 pjoin = os.path.join
7 7
8 8 from IPython.utils.path import get_ipython_dir
9 9 from IPython.utils.py3compat import PY3
10 10 from IPython.utils.traitlets import HasTraits, List, Unicode
11 11
12 12 USER_KERNEL_DIR = pjoin(get_ipython_dir(), 'kernels')
13 13
14 14 if os.name == 'nt':
15 15 programdata = os.environ.get('PROGRAMDATA', None)
16 16 if programdata:
17 17 SYSTEM_KERNEL_DIR = pjoin(programdata, 'ipython', 'kernels')
18 18 else: # PROGRAMDATA is not defined by default on XP.
19 19 SYSTEM_KERNEL_DIR = None
20 20 else:
21 21 SYSTEM_KERNEL_DIR = "/usr/share/ipython/kernels"
22 22
23 23 NATIVE_KERNEL_NAME = 'python3' if PY3 else 'python2'
24 24
25 25 class KernelSpec(HasTraits):
26 26 argv = List()
27 27 display_name = Unicode()
28 28 language = Unicode()
29 29 codemirror_mode = Unicode()
30 30
31 31 resource_dir = Unicode()
32 32
33 33 def __init__(self, resource_dir, argv, display_name, language,
34 34 codemirror_mode=None):
35 35 super(KernelSpec, self).__init__(resource_dir=resource_dir, argv=argv,
36 36 display_name=display_name, language=language,
37 37 codemirror_mode=codemirror_mode)
38 38 if not self.codemirror_mode:
39 39 self.codemirror_mode = self.language
40 40
41 41 @classmethod
42 42 def from_resource_dir(cls, resource_dir):
43 43 """Create a KernelSpec object by reading kernel.json
44 44
45 45 Pass the path to the *directory* containing kernel.json.
46 46 """
47 47 kernel_file = pjoin(resource_dir, 'kernel.json')
48 48 with io.open(kernel_file, 'r', encoding='utf-8') as f:
49 49 kernel_dict = json.load(f)
50 50 return cls(resource_dir=resource_dir, **kernel_dict)
51 51
52 52 def _is_kernel_dir(path):
53 53 """Is ``path`` a kernel directory?"""
54 54 return os.path.isdir(path) and os.path.isfile(pjoin(path, 'kernel.json'))
55 55
56 56 def _list_kernels_in(dir):
57 57 """Ensure dir exists, and return a mapping of kernel names to resource
58 58 directories from it.
59 59 """
60 60 if dir is None:
61 61 return {}
62 62 if not os.path.isdir(dir):
63 63 os.makedirs(dir, mode=0o644)
64 64 return {f.lower(): pjoin(dir, f) for f in os.listdir(dir)
65 65 if _is_kernel_dir(pjoin(dir, f))}
66 66
67 67 def _make_native_kernel_dir():
68 68 """Makes a kernel directory for the native kernel.
69 69
70 70 The native kernel is the kernel using the same Python runtime as this
71 71 process. This will put its informatino in the user kernels directory.
72 72 """
73 73 path = pjoin(USER_KERNEL_DIR, NATIVE_KERNEL_NAME)
74 74 os.mkdir(path)
75 75 with io.open(pjoin(path, 'kernel.json'), 'w', encoding='utf-8') as f:
76 76 json.dump({'argv':[NATIVE_KERNEL_NAME, '-c',
77 77 'from IPython.kernel.zmq.kernelapp import main; main()',
78 78 '-f', '{connection_file}'],
79 79 'display_name': 'Python 3' if PY3 else 'Python 2',
80 80 'language': 'python',
81 81 'codemirror_mode': {'name': 'python',
82 82 'version': sys.version_info[0]},
83 83 },
84 84 f)
85 85 # TODO: Copy icons into directory
86 86 return path
87 87
88 88 def list_kernel_specs():
89 89 """Returns a dict mapping kernel names to resource directories."""
90 90 d = _list_kernels_in(SYSTEM_KERNEL_DIR)
91 91 d.update(_list_kernels_in(USER_KERNEL_DIR))
92 92
93 93 if NATIVE_KERNEL_NAME not in d:
94 94 d[NATIVE_KERNEL_NAME] = _make_native_kernel_dir()
95 95 return d
96 96 # TODO: Caching?
97 97
98 98 def get_kernel_spec(kernel_name):
99 99 """Returns a :class:`KernelSpec` instance for the given kernel_name.
100 100
101 101 Raises KeyError if the given kernel name is not found.
102 102 """
103 if kernel_name == 'native':
104 kernel_name = NATIVE_KERNEL_NAME
103 105 d = list_kernel_specs()
104 106 resource_dir = d[kernel_name.lower()]
105 107 return KernelSpec.from_resource_dir(resource_dir) No newline at end of file
@@ -1,389 +1,410 b''
1 1 """Base class to manage a running kernel"""
2 2
3 3 #-----------------------------------------------------------------------------
4 4 # Copyright (C) 2013 The IPython Development Team
5 5 #
6 6 # Distributed under the terms of the BSD License. The full license is in
7 7 # the file COPYING, distributed as part of this software.
8 8 #-----------------------------------------------------------------------------
9 9
10 10 #-----------------------------------------------------------------------------
11 11 # Imports
12 12 #-----------------------------------------------------------------------------
13 13
14 14 from __future__ import absolute_import
15 15
16 16 # Standard library imports
17 17 import re
18 18 import signal
19 19 import sys
20 20 import time
21 import warnings
21 22
22 23 import zmq
23 24
24 25 # Local imports
25 26 from IPython.config.configurable import LoggingConfigurable
26 27 from IPython.utils.importstring import import_item
27 28 from IPython.utils.localinterfaces import is_local_ip, local_ips
28 29 from IPython.utils.traitlets import (
29 30 Any, Instance, Unicode, List, Bool, Type, DottedObjectName
30 31 )
31 32 from IPython.kernel import (
32 33 make_ipkernel_cmd,
33 34 launch_kernel,
35 kernelspec,
34 36 )
35 37 from .connect import ConnectionFileMixin
36 38 from .zmq.session import Session
37 39 from .managerabc import (
38 40 KernelManagerABC
39 41 )
40 42
41 43 #-----------------------------------------------------------------------------
42 44 # Main kernel manager class
43 45 #-----------------------------------------------------------------------------
44 46
45 47 class KernelManager(LoggingConfigurable, ConnectionFileMixin):
46 48 """Manages a single kernel in a subprocess on this host.
47 49
48 50 This version starts kernels with Popen.
49 51 """
50 52
51 53 # The PyZMQ Context to use for communication with the kernel.
52 54 context = Instance(zmq.Context)
53 55 def _context_default(self):
54 56 return zmq.Context.instance()
55 57
56 58 # The Session to use for communication with the kernel.
57 59 session = Instance(Session)
58 60 def _session_default(self):
59 61 return Session(parent=self)
60 62
61 63 # the class to create with our `client` method
62 64 client_class = DottedObjectName('IPython.kernel.blocking.BlockingKernelClient')
63 65 client_factory = Type()
64 66 def _client_class_changed(self, name, old, new):
65 67 self.client_factory = import_item(str(new))
66 68
67 69 # The kernel process with which the KernelManager is communicating.
68 70 # generally a Popen instance
69 71 kernel = Any()
72
73 kernel_name = Unicode('native')
74
75 kernel_spec = Instance(kernelspec.KernelSpec)
76
77 def _kernel_spec_default(self):
78 return kernelspec.get_kernel_spec(self.kernel_name)
79
80 def _kernel_name_changed(self, name, old, new):
81 self.kernel_spec = kernelspec.get_kernel_spec(new)
82 self.ipython_kernel = new in {'native', 'python2', 'python3'}
70 83
71 84 kernel_cmd = List(Unicode, config=True,
72 help="""The Popen Command to launch the kernel.
85 help="""DEPRECATED: Use kernel_name instead.
86
87 The Popen Command to launch the kernel.
73 88 Override this if you have a custom kernel.
74 89 If kernel_cmd is specified in a configuration file,
75 90 IPython does not pass any arguments to the kernel,
76 91 because it cannot make any assumptions about the
77 92 arguments that the kernel understands. In particular,
78 93 this means that the kernel does not receive the
79 94 option --debug if it given on the IPython command line.
80 95 """
81 96 )
82 97
83 98 def _kernel_cmd_changed(self, name, old, new):
99 warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to "
100 "start different kernels.")
84 101 self.ipython_kernel = False
85 102
86 103 ipython_kernel = Bool(True)
87 104
88 105 # Protected traits
89 106 _launch_args = Any()
90 107 _control_socket = Any()
91 108
92 109 _restarter = Any()
93 110
94 111 autorestart = Bool(False, config=True,
95 112 help="""Should we autorestart the kernel if it dies."""
96 113 )
97 114
98 115 def __del__(self):
99 116 self._close_control_socket()
100 117 self.cleanup_connection_file()
101 118
102 119 #--------------------------------------------------------------------------
103 120 # Kernel restarter
104 121 #--------------------------------------------------------------------------
105 122
106 123 def start_restarter(self):
107 124 pass
108 125
109 126 def stop_restarter(self):
110 127 pass
111 128
112 129 def add_restart_callback(self, callback, event='restart'):
113 130 """register a callback to be called when a kernel is restarted"""
114 131 if self._restarter is None:
115 132 return
116 133 self._restarter.add_callback(callback, event)
117 134
118 135 def remove_restart_callback(self, callback, event='restart'):
119 136 """unregister a callback to be called when a kernel is restarted"""
120 137 if self._restarter is None:
121 138 return
122 139 self._restarter.remove_callback(callback, event)
123 140
124 141 #--------------------------------------------------------------------------
125 142 # create a Client connected to our Kernel
126 143 #--------------------------------------------------------------------------
127 144
128 145 def client(self, **kwargs):
129 146 """Create a client configured to connect to our kernel"""
130 147 if self.client_factory is None:
131 148 self.client_factory = import_item(self.client_class)
132 149
133 150 kw = {}
134 151 kw.update(self.get_connection_info())
135 152 kw.update(dict(
136 153 connection_file=self.connection_file,
137 154 session=self.session,
138 155 parent=self,
139 156 ))
140 157
141 158 # add kwargs last, for manual overrides
142 159 kw.update(kwargs)
143 160 return self.client_factory(**kw)
144 161
145 162 #--------------------------------------------------------------------------
146 163 # Kernel management
147 164 #--------------------------------------------------------------------------
148 165
149 166 def format_kernel_cmd(self, **kw):
150 167 """replace templated args (e.g. {connection_file})"""
151 168 if self.kernel_cmd:
152 169 cmd = self.kernel_cmd
153 else:
170 elif self.kernel_name == 'native':
171 # The native kernel gets special handling
154 172 cmd = make_ipkernel_cmd(
155 173 'from IPython.kernel.zmq.kernelapp import main; main()',
156 174 **kw
157 175 )
176 else:
177 cmd = self.kernel_spec.argv
178
158 179 ns = dict(connection_file=self.connection_file)
159 180 ns.update(self._launch_args)
160 181
161 182 pat = re.compile(r'\{([A-Za-z0-9_]+)\}')
162 183 def from_ns(match):
163 184 """Get the key out of ns if it's there, otherwise no change."""
164 185 return ns.get(match.group(1), match.group())
165 186
166 187 return [ pat.sub(from_ns, arg) for arg in cmd ]
167 188
168 189 def _launch_kernel(self, kernel_cmd, **kw):
169 190 """actually launch the kernel
170 191
171 192 override in a subclass to launch kernel subprocesses differently
172 193 """
173 194 return launch_kernel(kernel_cmd, **kw)
174 195
175 196 # Control socket used for polite kernel shutdown
176 197
177 198 def _connect_control_socket(self):
178 199 if self._control_socket is None:
179 200 self._control_socket = self.connect_control()
180 201 self._control_socket.linger = 100
181 202
182 203 def _close_control_socket(self):
183 204 if self._control_socket is None:
184 205 return
185 206 self._control_socket.close()
186 207 self._control_socket = None
187 208
188 209 def start_kernel(self, **kw):
189 210 """Starts a kernel on this host in a separate process.
190 211
191 212 If random ports (port=0) are being used, this method must be called
192 213 before the channels are created.
193 214
194 215 Parameters
195 216 ----------
196 217 **kw : optional
197 218 keyword arguments that are passed down to build the kernel_cmd
198 219 and launching the kernel (e.g. Popen kwargs).
199 220 """
200 221 if self.transport == 'tcp' and not is_local_ip(self.ip):
201 222 raise RuntimeError("Can only launch a kernel on a local interface. "
202 223 "Make sure that the '*_address' attributes are "
203 224 "configured properly. "
204 225 "Currently valid addresses are: %s" % local_ips()
205 226 )
206 227
207 228 # write connection file / get default ports
208 229 self.write_connection_file()
209 230
210 231 # save kwargs for use in restart
211 232 self._launch_args = kw.copy()
212 233 # build the Popen cmd
213 234 kernel_cmd = self.format_kernel_cmd(**kw)
214 235 # launch the kernel subprocess
215 236 self.kernel = self._launch_kernel(kernel_cmd,
216 237 ipython_kernel=self.ipython_kernel,
217 238 **kw)
218 239 self.start_restarter()
219 240 self._connect_control_socket()
220 241
221 242 def _send_shutdown_request(self, restart=False):
222 243 """TODO: send a shutdown request via control channel"""
223 244 content = dict(restart=restart)
224 245 msg = self.session.msg("shutdown_request", content=content)
225 246 self.session.send(self._control_socket, msg)
226 247
227 248 def shutdown_kernel(self, now=False, restart=False):
228 249 """Attempts to the stop the kernel process cleanly.
229 250
230 251 This attempts to shutdown the kernels cleanly by:
231 252
232 253 1. Sending it a shutdown message over the shell channel.
233 254 2. If that fails, the kernel is shutdown forcibly by sending it
234 255 a signal.
235 256
236 257 Parameters
237 258 ----------
238 259 now : bool
239 260 Should the kernel be forcible killed *now*. This skips the
240 261 first, nice shutdown attempt.
241 262 restart: bool
242 263 Will this kernel be restarted after it is shutdown. When this
243 264 is True, connection files will not be cleaned up.
244 265 """
245 266 # Stop monitoring for restarting while we shutdown.
246 267 self.stop_restarter()
247 268
248 269 # FIXME: Shutdown does not work on Windows due to ZMQ errors!
249 270 if now or sys.platform == 'win32':
250 271 if self.has_kernel:
251 272 self._kill_kernel()
252 273 else:
253 274 # Don't send any additional kernel kill messages immediately, to give
254 275 # the kernel a chance to properly execute shutdown actions. Wait for at
255 276 # most 1s, checking every 0.1s.
256 277 self._send_shutdown_request(restart=restart)
257 278 for i in range(10):
258 279 if self.is_alive():
259 280 time.sleep(0.1)
260 281 else:
261 282 break
262 283 else:
263 284 # OK, we've waited long enough.
264 285 if self.has_kernel:
265 286 self._kill_kernel()
266 287
267 288 if not restart:
268 289 self.cleanup_connection_file()
269 290 self.cleanup_ipc_files()
270 291 else:
271 292 self.cleanup_ipc_files()
272 293
273 294 self._close_control_socket()
274 295
275 296 def restart_kernel(self, now=False, **kw):
276 297 """Restarts a kernel with the arguments that were used to launch it.
277 298
278 299 If the old kernel was launched with random ports, the same ports will be
279 300 used for the new kernel. The same connection file is used again.
280 301
281 302 Parameters
282 303 ----------
283 304 now : bool, optional
284 305 If True, the kernel is forcefully restarted *immediately*, without
285 306 having a chance to do any cleanup action. Otherwise the kernel is
286 307 given 1s to clean up before a forceful restart is issued.
287 308
288 309 In all cases the kernel is restarted, the only difference is whether
289 310 it is given a chance to perform a clean shutdown or not.
290 311
291 312 **kw : optional
292 313 Any options specified here will overwrite those used to launch the
293 314 kernel.
294 315 """
295 316 if self._launch_args is None:
296 317 raise RuntimeError("Cannot restart the kernel. "
297 318 "No previous call to 'start_kernel'.")
298 319 else:
299 320 # Stop currently running kernel.
300 321 self.shutdown_kernel(now=now, restart=True)
301 322
302 323 # Start new kernel.
303 324 self._launch_args.update(kw)
304 325 self.start_kernel(**self._launch_args)
305 326
306 327 # FIXME: Messages get dropped in Windows due to probable ZMQ bug
307 328 # unless there is some delay here.
308 329 if sys.platform == 'win32':
309 330 time.sleep(0.2)
310 331
311 332 @property
312 333 def has_kernel(self):
313 334 """Has a kernel been started that we are managing."""
314 335 return self.kernel is not None
315 336
316 337 def _kill_kernel(self):
317 338 """Kill the running kernel.
318 339
319 340 This is a private method, callers should use shutdown_kernel(now=True).
320 341 """
321 342 if self.has_kernel:
322 343
323 344 # Signal the kernel to terminate (sends SIGKILL on Unix and calls
324 345 # TerminateProcess() on Win32).
325 346 try:
326 347 self.kernel.kill()
327 348 except OSError as e:
328 349 # In Windows, we will get an Access Denied error if the process
329 350 # has already terminated. Ignore it.
330 351 if sys.platform == 'win32':
331 352 if e.winerror != 5:
332 353 raise
333 354 # On Unix, we may get an ESRCH error if the process has already
334 355 # terminated. Ignore it.
335 356 else:
336 357 from errno import ESRCH
337 358 if e.errno != ESRCH:
338 359 raise
339 360
340 361 # Block until the kernel terminates.
341 362 self.kernel.wait()
342 363 self.kernel = None
343 364 else:
344 365 raise RuntimeError("Cannot kill kernel. No kernel is running!")
345 366
346 367 def interrupt_kernel(self):
347 368 """Interrupts the kernel by sending it a signal.
348 369
349 370 Unlike ``signal_kernel``, this operation is well supported on all
350 371 platforms.
351 372 """
352 373 if self.has_kernel:
353 374 if sys.platform == 'win32':
354 375 from .zmq.parentpoller import ParentPollerWindows as Poller
355 376 Poller.send_interrupt(self.kernel.win32_interrupt_event)
356 377 else:
357 378 self.kernel.send_signal(signal.SIGINT)
358 379 else:
359 380 raise RuntimeError("Cannot interrupt kernel. No kernel is running!")
360 381
361 382 def signal_kernel(self, signum):
362 383 """Sends a signal to the kernel.
363 384
364 385 Note that since only SIGTERM is supported on Windows, this function is
365 386 only useful on Unix systems.
366 387 """
367 388 if self.has_kernel:
368 389 self.kernel.send_signal(signum)
369 390 else:
370 391 raise RuntimeError("Cannot signal kernel. No kernel is running!")
371 392
372 393 def is_alive(self):
373 394 """Is the kernel process still running?"""
374 395 if self.has_kernel:
375 396 if self.kernel.poll() is None:
376 397 return True
377 398 else:
378 399 return False
379 400 else:
380 401 # we don't have a kernel
381 402 return False
382 403
383 404
384 405 #-----------------------------------------------------------------------------
385 406 # ABC Registration
386 407 #-----------------------------------------------------------------------------
387 408
388 409 KernelManagerABC.register(KernelManager)
389 410
General Comments 0
You need to be logged in to leave comments. Login now