From 03eb23cd71566dd118cb5eb6afa427e1c57b2ce9 2011-08-07 04:33:55 From: Brian E. Granger Date: 2011-08-07 04:33:55 Subject: [PATCH] Adding kernel/notebook associations. --- diff --git a/IPython/frontend/html/notebook/handlers.py b/IPython/frontend/html/notebook/handlers.py index b6cb9f7..ab680c4 100644 --- a/IPython/frontend/html/notebook/handlers.py +++ b/IPython/frontend/html/notebook/handlers.py @@ -13,7 +13,7 @@ from tornado import websocket #----------------------------------------------------------------------------- -# Handlers +# Top-level handlers #----------------------------------------------------------------------------- @@ -38,25 +38,45 @@ class NamedNotebookHandler(web.RequestHandler): self.render('notebook.html', notebook_id=notebook_id) -class KernelHandler(web.RequestHandler): +#----------------------------------------------------------------------------- +# Kernel handlers +#----------------------------------------------------------------------------- + + +class MainKernelHandler(web.RequestHandler): def get(self): - self.finish(json.dumps(self.application.kernel_ids)) + rkm = self.application.routing_kernel_manager + self.finish(json.dumps(rkm.kernel_ids)) def post(self): - kernel_id = self.application.start_kernel() + rkm = self.application.routing_kernel_manager + notebook_id = self.get_argument('notebook', default=None) + kernel_id = rkm.start_kernel(notebook_id) self.set_header('Location', '/'+kernel_id) self.finish(json.dumps(kernel_id)) +class KernelHandler(web.RequestHandler): + + SUPPORTED_METHODS = ('DELETE') + + def delete(self, kernel_id): + rkm = self.application.routing_kernel_manager + self.kill_kernel(kernel_id) + self.set_status(204) + self.finish() + + class KernelActionHandler(web.RequestHandler): def post(self, kernel_id, action): - # TODO: figure out a better way of handling RPC style calls. + rkm = self.application.routing_kernel_manager if action == 'interrupt': - self.application.interrupt_kernel(kernel_id) + rkm.interrupt_kernel(kernel_id) + self.set_status(204) if action == 'restart': - new_kernel_id = self.application.restart_kernel(kernel_id) + new_kernel_id = rkm.restart_kernel(kernel_id) self.write(json.dumps(new_kernel_id)) self.finish() @@ -67,7 +87,8 @@ class ZMQStreamHandler(websocket.WebSocketHandler): self.stream_name = stream_name def open(self, kernel_id): - self.router = self.application.get_router(kernel_id, self.stream_name) + rkm = self.application.routing_kernel_manager + self.router = rkm.get_router(kernel_id, self.stream_name) self.client_id = self.router.register_client(self) logging.info("Connection open: %s, %s" % (kernel_id, self.client_id)) @@ -79,6 +100,10 @@ class ZMQStreamHandler(websocket.WebSocketHandler): logging.info("Connection closed: %s" % self.client_id) +#----------------------------------------------------------------------------- +# Notebook web service handlers +#----------------------------------------------------------------------------- + class NotebookRootHandler(web.RequestHandler): def get(self): diff --git a/IPython/frontend/html/notebook/kernelmanager.py b/IPython/frontend/html/notebook/kernelmanager.py index 9a1fe37..f5b2ecd 100644 --- a/IPython/frontend/html/notebook/kernelmanager.py +++ b/IPython/frontend/html/notebook/kernelmanager.py @@ -17,9 +17,13 @@ import uuid import zmq +from tornado import web + +from .routers import IOPubStreamRouter, ShellStreamRouter + from IPython.config.configurable import LoggingConfigurable from IPython.zmq.ipkernel import launch_kernel -from IPython.utils.traitlets import Instance, Dict +from IPython.utils.traitlets import Instance, Dict, List, Unicode #----------------------------------------------------------------------------- # Classes @@ -55,7 +59,7 @@ class KernelManager(LoggingConfigurable): def start_kernel(self, **kwargs): """Start a new kernel.""" - kernel_id = str(uuid.uuid4()) + kernel_id = unicode(uuid.uuid4()) (process, shell_port, iopub_port, stdin_port, hb_port) = launch_kernel(**kwargs) # Store the information for contacting the kernel. This assumes the kernel is # running on localhost. @@ -186,3 +190,117 @@ class KernelManager(LoggingConfigurable): config=self.config, context=self.context, log=self.log ) + +class RoutingKernelManager(LoggingConfigurable): + """A KernelManager that handles WebSocket routing and HTTP error handling""" + + kernel_argv = List(Unicode) + kernel_manager = Instance(KernelManager) + + _routers = Dict() + _session_dict = Dict() + _notebook_mapping = Dict() + + #------------------------------------------------------------------------- + # Methods for managing kernels and sessions + #------------------------------------------------------------------------- + + @property + def kernel_ids(self): + return self.kernel_manager.kernel_ids + + def notebook_for_kernel(self, kernel_id): + notebook_ids = [k for k, v in self._notebook_mapping.iteritems() if v == kernel_id] + if len(notebook_ids) == 1: + return notebook_ids[0] + else: + return None + + def delete_mapping_for_kernel(self, kernel_id): + notebook_id = self.notebook_for_kernel(kernel_id) + if notebook_id is not None: + del self._notebook_mapping[notebook_id] + + def start_kernel(self, notebook_id=None): + self.log.info + kernel_id = self._notebook_mapping.get(notebook_id) + if kernel_id is None: + kwargs = dict() + kwargs['extra_arguments'] = self.kernel_argv + kernel_id = self.kernel_manager.start_kernel(**kwargs) + if notebook_id is not None: + self._notebook_mapping[notebook_id] = kernel_id + self.log.info("Kernel started for notebook %s: %s" % (notebook_id,kernel_id)) + self.log.debug("Kernel args: %r" % kwargs) + self.start_session_manager(kernel_id) + else: + self.log.info("Using existing kernel: %s" % kernel_id) + return kernel_id + + def start_session_manager(self, kernel_id): + sm = self.kernel_manager.create_session_manager(kernel_id) + self._session_dict[kernel_id] = sm + iopub_stream = sm.get_iopub_stream() + shell_stream = sm.get_shell_stream() + iopub_router = IOPubStreamRouter( + zmq_stream=iopub_stream, session=sm.session, config=self.config + ) + shell_router = ShellStreamRouter( + zmq_stream=shell_stream, session=sm.session, config=self.config + ) + self._routers[(kernel_id, 'iopub')] = iopub_router + self._routers[(kernel_id, 'shell')] = shell_router + + def kill_kernel(self, kernel_id): + if kernel_id not in self.kernel_manager: + raise web.HTTPError(404) + try: + sm = self._session_dict.pop(kernel_id) + except KeyError: + raise web.HTTPError(404) + sm.stop() + self.kernel_manager.kill_kernel(kernel_id) + self.delete_mapping_for_kernel(kernel_id) + self.log.info("Kernel killed: %s" % kernel_id) + + def interrupt_kernel(self, kernel_id): + if kernel_id not in self.kernel_manager: + raise web.HTTPError(404) + self.kernel_manager.interrupt_kernel(kernel_id) + self.log.debug("Kernel interrupted: %s" % kernel_id) + + def restart_kernel(self, kernel_id): + if kernel_id not in self.kernel_manager: + raise web.HTTPError(404) + + # Get the notebook_id to preserve the kernel/notebook association + notebook_id = self.notebook_for_kernel(kernel_id) + # Create the new kernel first so we can move the clients over. + new_kernel_id = self.start_kernel() + + # Copy the clients over to the new routers. + old_iopub_router = self.get_router(kernel_id, 'iopub') + old_shell_router = self.get_router(kernel_id, 'shell') + new_iopub_router = self.get_router(new_kernel_id, 'iopub') + new_shell_router = self.get_router(new_kernel_id, 'shell') + new_iopub_router.copy_clients(old_iopub_router) + new_shell_router.copy_clients(old_shell_router) + + # Now shutdown the old session and the kernel. + # TODO: This causes a hard crash in ZMQStream.close, which sets + # self.socket to None to hastily. We will need to fix this in PyZMQ + # itself. For now, we just leave the old kernel running :( + # Maybe this is fixed now, but nothing was changed really. + self.kill_kernel(kernel_id) + + # Now save the new kernel/notebook association. We have to save it + # after the old kernel is killed as that will delete the mapping. + self._notebook_mapping[notebook_id] = kernel_id + + self.log.debug("Kernel restarted: %s -> %s" % (kernel_id, new_kernel_id)) + return new_kernel_id + + def get_router(self, kernel_id, stream_name): + router = self._routers[(kernel_id, stream_name)] + return router + diff --git a/IPython/frontend/html/notebook/notebookapp.py b/IPython/frontend/html/notebook/notebookapp.py index a8314bb..4b59303 100644 --- a/IPython/frontend/html/notebook/notebookapp.py +++ b/IPython/frontend/html/notebook/notebookapp.py @@ -27,14 +27,13 @@ tornado.ioloop = ioloop from tornado import httpserver from tornado import web -from .kernelmanager import KernelManager +from .kernelmanager import KernelManager, RoutingKernelManager from .sessionmanager import SessionManager from .handlers import ( NBBrowserHandler, NewHandler, NamedNotebookHandler, - KernelHandler, KernelActionHandler, ZMQStreamHandler, + MainKernelHandler, KernelHandler, KernelActionHandler, ZMQStreamHandler, NotebookRootHandler, NotebookHandler ) -from .routers import IOPubStreamRouter, ShellStreamRouter from .notebookmanager import NotebookManager from IPython.core.application import BaseIPythonApplication @@ -65,12 +64,13 @@ LOCALHOST = '127.0.0.1' class NotebookWebApplication(web.Application): - def __init__(self, kernel_manager, log, kernel_argv, config): + def __init__(self, routing_kernel_manager, notebook_manager, log): handlers = [ (r"/", NBBrowserHandler), (r"/new", NewHandler), (r"/%s" % _notebook_id_regex, NamedNotebookHandler), - (r"/kernels", KernelHandler), + (r"/kernels", MainKernelHandler), + (r"/kernels/%s" % _kernel_id_regex, KernelHandler), (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler), (r"/kernels/%s/iopub" % _kernel_id_regex, ZMQStreamHandler, dict(stream_name='iopub')), (r"/kernels/%s/shell" % _kernel_id_regex, ZMQStreamHandler, dict(stream_name='shell')), @@ -83,82 +83,10 @@ class NotebookWebApplication(web.Application): ) web.Application.__init__(self, handlers, **settings) - self.kernel_manager = kernel_manager + self.routing_kernel_manager = routing_kernel_manager self.log = log - self.kernel_argv = kernel_argv - self.config = config - self._routers = {} - self._session_dict = {} - self.notebook_manager = NotebookManager(config=self.config) - - #------------------------------------------------------------------------- - # Methods for managing kernels and sessions - #------------------------------------------------------------------------- - - @property - def kernel_ids(self): - return self.kernel_manager.kernel_ids - - def start_kernel(self): - kwargs = dict() - kwargs['extra_arguments'] = self.kernel_argv - kernel_id = self.kernel_manager.start_kernel(**kwargs) - self.log.info("Kernel started: %s" % kernel_id) - self.log.debug("Kernel args: %r" % kwargs) - self.start_session_manager(kernel_id) - return kernel_id - - def start_session_manager(self, kernel_id): - sm = self.kernel_manager.create_session_manager(kernel_id) - self._session_dict[kernel_id] = sm - iopub_stream = sm.get_iopub_stream() - shell_stream = sm.get_shell_stream() - iopub_router = IOPubStreamRouter( - zmq_stream=iopub_stream, session=sm.session, config=self.config - ) - shell_router = ShellStreamRouter( - zmq_stream=shell_stream, session=sm.session, config=self.config - ) - self._routers[(kernel_id, 'iopub')] = iopub_router - self._routers[(kernel_id, 'shell')] = shell_router - - def kill_kernel(self, kernel_id): - sm = self._session_dict.pop(kernel_id) - sm.stop() - self.kernel_manager.kill_kernel(kernel_id) - self.log.info("Kernel killed: %s" % kernel_id) - - def interrupt_kernel(self, kernel_id): - self.kernel_manager.interrupt_kernel(kernel_id) - self.log.debug("Kernel interrupted: %s" % kernel_id) - - def restart_kernel(self, kernel_id): - # Create the new kernel first so we can move the clients over. - new_kernel_id = self.start_kernel() - - # Copy the clients over to the new routers. - old_iopub_router = self.get_router(kernel_id, 'iopub') - old_shell_router = self.get_router(kernel_id, 'shell') - new_iopub_router = self.get_router(new_kernel_id, 'iopub') - new_shell_router = self.get_router(new_kernel_id, 'shell') - new_iopub_router.copy_clients(old_iopub_router) - new_shell_router.copy_clients(old_shell_router) - - # Now shutdown the old session and the kernel. - # TODO: This causes a hard crash in ZMQStream.close, which sets - # self.socket to None to hastily. We will need to fix this in PyZMQ - # itself. For now, we just leave the old kernel running :( - # Maybe this is fixed now, but nothing was changed really. - self.kill_kernel(kernel_id) - - self.log.debug("Kernel restarted: %s -> %s" % (kernel_id, new_kernel_id)) - return new_kernel_id - - def get_router(self, kernel_id, stream_name): - router = self._routers[(kernel_id, stream_name)] - return router + self.notebook_manager = notebook_manager - #----------------------------------------------------------------------------- # Aliases and Flags @@ -196,6 +124,7 @@ class IPythonNotebookApp(BaseIPythonApplication): """ classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session, + RoutingKernelManager, NotebookManager, KernelManager, SessionManager, RichIPythonWidget] flags = Dict(flags) aliases = Dict(aliases) @@ -232,12 +161,16 @@ class IPythonNotebookApp(BaseIPythonApplication): if a.startswith('-') and a.lstrip('-') in notebook_flags: self.kernel_argv.remove(a) - def init_kernel_manager(self): + def init_configurables(self): # Don't let Qt or ZMQ swallow KeyboardInterupts. signal.signal(signal.SIGINT, signal.SIG_DFL) # Create a KernelManager and start a kernel. self.kernel_manager = KernelManager(config=self.config, log=self.log) + self.routing_kernel_manager = RoutingKernelManager(config=self.config, log=self.log, + kernel_manager=self.kernel_manager, kernel_argv=self.kernel_argv + ) + self.notebook_manager = NotebookManager(config=self.config, log=self.log) def init_logging(self): super(IPythonNotebookApp, self).init_logging() @@ -248,9 +181,9 @@ class IPythonNotebookApp(BaseIPythonApplication): def initialize(self, argv=None): super(IPythonNotebookApp, self).initialize(argv) - self.init_kernel_manager() + self.init_configurables() self.web_app = NotebookWebApplication( - self.kernel_manager, self.log, self.kernel_argv, self.config + self.routing_kernel_manager, self.notebook_manager, self.log ) self.http_server = httpserver.HTTPServer(self.web_app) self.http_server.listen(self.port) diff --git a/IPython/frontend/html/notebook/notebookmanager.py b/IPython/frontend/html/notebook/notebookmanager.py index d66d359..e8f8d10 100644 --- a/IPython/frontend/html/notebook/notebookmanager.py +++ b/IPython/frontend/html/notebook/notebookmanager.py @@ -15,7 +15,7 @@ import uuid from tornado import web -from IPython.config.configurable import Configurable +from IPython.config.configurable import LoggingConfigurable from IPython.nbformat import current from IPython.utils.traitlets import Unicode, List, Dict @@ -25,7 +25,7 @@ from IPython.utils.traitlets import Unicode, List, Dict #----------------------------------------------------------------------------- -class NotebookManager(Configurable): +class NotebookManager(LoggingConfigurable): notebook_dir = Unicode(os.getcwd()) filename_ext = Unicode(u'.ipynb') diff --git a/IPython/frontend/html/notebook/static/js/kernel.js b/IPython/frontend/html/notebook/static/js/kernel.js index e7f9f9b..44a5a6d 100644 --- a/IPython/frontend/html/notebook/static/js/kernel.js +++ b/IPython/frontend/html/notebook/static/js/kernel.js @@ -28,9 +28,10 @@ var IPython = (function (IPython) { return msg; } - Kernel.prototype.start_kernel = function (callback) { + Kernel.prototype.start_kernel = function (notebook_id, callback) { var that = this; - $.post(this.base_url, + var qs = $.param({notebook:notebook_id}); + $.post(this.base_url + '?' + qs, function (kernel_id) { that._handle_start_kernel(kernel_id, callback); }, diff --git a/IPython/frontend/html/notebook/static/js/notebook.js b/IPython/frontend/html/notebook/static/js/notebook.js index 2db0cec..999c85c 100644 --- a/IPython/frontend/html/notebook/static/js/notebook.js +++ b/IPython/frontend/html/notebook/static/js/notebook.js @@ -391,7 +391,8 @@ var IPython = (function (IPython) { Notebook.prototype.start_kernel = function () { this.kernel = new IPython.Kernel(); - this.kernel.start_kernel($.proxy(this.kernel_started, this)); + var notebook_id = IPython.save_widget.get_notebook_id(); + this.kernel.start_kernel(notebook_id, $.proxy(this.kernel_started, this)); };