Merge pull request #5598 from takluyver/kernelspec

Kernel registry (IPEP 25)

IPython/kernel/kernelspec.py

@@ -0,0 +1,142 b''
+import io
+import json
+import os
+import sys
+
+pjoin = os.path.join
+
+from IPython.utils.path import get_ipython_dir
+from IPython.utils.py3compat import PY3
+from IPython.utils.traitlets import HasTraits, List, Unicode, Dict
+
+if os.name == 'nt':
+    programdata = os.environ.get('PROGRAMDATA', None)
+    if programdata:
+        SYSTEM_KERNEL_DIR = pjoin(programdata, 'ipython', 'kernels')
+    else:  # PROGRAMDATA is not defined by default on XP.
+        SYSTEM_KERNEL_DIR = None
+else:
+    SYSTEM_KERNEL_DIR = "/usr/share/ipython/kernels"
+    
+NATIVE_KERNEL_NAME = 'python3' if PY3 else 'python2'
+
+class KernelSpec(HasTraits):
+    argv = List()
+    display_name = Unicode()
+    language = Unicode()
+    codemirror_mode = None
+    env = Dict()
+    
+    resource_dir = Unicode()
+    
+    def __init__(self, resource_dir, argv, display_name, language,
+                 codemirror_mode=None):
+        super(KernelSpec, self).__init__(resource_dir=resource_dir, argv=argv,
+                display_name=display_name, language=language,
+                codemirror_mode=codemirror_mode)
+        if not self.codemirror_mode:
+            self.codemirror_mode = self.language
+    
+    @classmethod
+    def from_resource_dir(cls, resource_dir):
+        """Create a KernelSpec object by reading kernel.json
+        
+        Pass the path to the *directory* containing kernel.json.
+        """
+        kernel_file = pjoin(resource_dir, 'kernel.json')
+        with io.open(kernel_file, 'r', encoding='utf-8') as f:
+            kernel_dict = json.load(f)
+        return cls(resource_dir=resource_dir, **kernel_dict)
+
+def _is_kernel_dir(path):
+    """Is ``path`` a kernel directory?"""
+    return os.path.isdir(path) and os.path.isfile(pjoin(path, 'kernel.json'))
+
+def _list_kernels_in(dir):
+    """Return a mapping of kernel names to resource directories from dir.
+    
+    If dir is None or does not exist, returns an empty dict.
+    """
+    if dir is None or not os.path.isdir(dir):
+        return {}
+    return {f.lower(): pjoin(dir, f) for f in os.listdir(dir)
+                        if _is_kernel_dir(pjoin(dir, f))}
+
+class NoSuchKernel(KeyError):
+    def __init__(self, name):
+        self.name = name
+
+class KernelSpecManager(HasTraits):
+    ipython_dir = Unicode()
+    def _ipython_dir_default(self):
+        return get_ipython_dir()
+
+    user_kernel_dir = Unicode()
+    def _user_kernel_dir_default(self):
+        return pjoin(self.ipython_dir, 'kernels')
+    
+    kernel_dirs = List(
+        help="List of kernel directories to search. Later ones take priority over earlier."    
+    )    
+    def _kernel_dirs_default(self):
+        return [
+            SYSTEM_KERNEL_DIR,
+            self.user_kernel_dir,
+        ]
+
+    def _make_native_kernel_dir(self):
+        """Makes a kernel directory for the native kernel.
+        
+        The native kernel is the kernel using the same Python runtime as this
+        process. This will put its informatino in the user kernels directory.
+        """
+        path = pjoin(self.user_kernel_dir, NATIVE_KERNEL_NAME)
+        os.makedirs(path, mode=0o755)
+        with open(pjoin(path, 'kernel.json'), 'w') as f:
+            json.dump({'argv':[NATIVE_KERNEL_NAME, '-c',
+                               'from IPython.kernel.zmq.kernelapp import main; main()',
+                                '-f', '{connection_file}'],
+                       'display_name': 'Python 3' if PY3 else 'Python 2',
+                       'language': 'python',
+                       'codemirror_mode': {'name': 'python',
+                                           'version': sys.version_info[0]},
+                      },
+                      f, indent=1)
+        # TODO: Copy icons into directory
+        return path
+    
+    def find_kernel_specs(self):
+        """Returns a dict mapping kernel names to resource directories."""
+        d = {}
+        for kernel_dir in self.kernel_dirs:
+            d.update(_list_kernels_in(kernel_dir))
+        
+        if NATIVE_KERNEL_NAME not in d:
+            d[NATIVE_KERNEL_NAME] = self._make_native_kernel_dir()
+        return d
+        # TODO: Caching?
+    
+    def get_kernel_spec(self, kernel_name):
+        """Returns a :class:`KernelSpec` instance for the given kernel_name.
+        
+        Raises :exc:`NoSuchKernel` if the given kernel name is not found.
+        """
+        if kernel_name == 'python':
+            kernel_name = NATIVE_KERNEL_NAME
+        d = self.find_kernel_specs()
+        try:
+            resource_dir = d[kernel_name.lower()]
+        except KeyError:
+            raise NoSuchKernel(kernel_name)
+        return KernelSpec.from_resource_dir(resource_dir)
+
+def find_kernel_specs():
+    """Returns a dict mapping kernel names to resource directories."""
+    return KernelSpecManager().find_kernel_specs()
+
+def get_kernel_spec(kernel_name):
+    """Returns a :class:`KernelSpec` instance for the given kernel_name.
+    
+    Raises KeyError if the given kernel name is not found.
+    """
+    return KernelSpecManager().get_kernel_spec(kernel_name)
  No newline at end of file

IPython/kernel/tests/test_kernelspec.py

@@ -0,0 +1,39 b''
+import json
+import os
+from os.path import join as pjoin
+import unittest
+
+from IPython.utils.tempdir import TemporaryDirectory
+from IPython.kernel import kernelspec
+
+sample_kernel_json = {'argv':['cat', '{connection_file}'],
+                      'display_name':'Test kernel',
+                      'language':'bash',
+                     }
+
+class KernelSpecTests(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = td = TemporaryDirectory()
+        self.sample_kernel_dir = pjoin(td.name, 'kernels', 'Sample')
+        os.makedirs(self.sample_kernel_dir)
+        json_file = pjoin(self.sample_kernel_dir, 'kernel.json')
+        with open(json_file, 'w') as f:
+            json.dump(sample_kernel_json, f)
+
+        self.ksm = kernelspec.KernelSpecManager(ipython_dir=td.name)
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+    def test_find_kernel_specs(self):
+        kernels = self.ksm.find_kernel_specs()
+        self.assertEqual(kernels['sample'], self.sample_kernel_dir)
+
+    def test_get_kernel_spec(self):
+        ks = self.ksm.get_kernel_spec('SAMPLE')  # Case insensitive
+        self.assertEqual(ks.resource_dir, self.sample_kernel_dir)
+        self.assertEqual(ks.argv, sample_kernel_json['argv'])
+        self.assertEqual(ks.display_name, sample_kernel_json['display_name'])
+        self.assertEqual(ks.language, sample_kernel_json['language'])
+        self.assertEqual(ks.codemirror_mode, sample_kernel_json['language'])
+        self.assertEqual(ks.env, {})
  No newline at end of file

IPython/consoleapp.py

@@ -35,6 +35,7 b' from IPython.core.profiledir import ProfileDir'
 from IPython.kernel.blocking import BlockingKernelClient
 from IPython.kernel import KernelManager
 from IPython.kernel import tunnel_to_kernel, find_connection_file, swallow_argv
+from IPython.kernel.kernelspec import NoSuchKernel
 from IPython.utils.path import filefind
 from IPython.utils.py3compat import str_to_bytes
 from IPython.utils.traitlets import (
@@ -98,6 +99,7 b' app_aliases = dict('
     existing = 'IPythonConsoleApp.existing',
     f = 'IPythonConsoleApp.connection_file',
 
+    kernel = 'IPythonConsoleApp.kernel_name',
 
     ssh = 'IPythonConsoleApp.sshserver',
 )
@@ -174,6 +176,9 b' class IPythonConsoleApp(ConnectionFileMixin):'
     existing = CUnicode('', config=True,
         help="""Connect to an already running kernel""")
 
+    kernel_name = Unicode('python', config=True,
+        help="""The name of the default kernel to start.""")
+
     confirm_exit = CBool(True, config=True,
         help="""
         Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
@@ -327,16 +332,23 b' class IPythonConsoleApp(ConnectionFileMixin):'
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
         # Create a KernelManager and start a kernel.
-        self.kernel_manager = self.kernel_manager_class(
-                                ip=self.ip,
-                                transport=self.transport,
-                                shell_port=self.shell_port,
-                                iopub_port=self.iopub_port,
-                                stdin_port=self.stdin_port,
-                                hb_port=self.hb_port,
-                                connection_file=self.connection_file,
-                                parent=self,
-        )
+        try:
+            self.kernel_manager = self.kernel_manager_class(
+                                    ip=self.ip,
+                                    transport=self.transport,
+                                    shell_port=self.shell_port,
+                                    iopub_port=self.iopub_port,
+                                    stdin_port=self.stdin_port,
+                                    hb_port=self.hb_port,
+                                    connection_file=self.connection_file,
+                                    kernel_name=self.kernel_name,
+                                    parent=self,
+                                    ipython_dir=self.ipython_dir,
+            )
+        except NoSuchKernel:
+            self.log.critical("Could not find kernel %s", self.kernel_name)
+            self.exit(1)
+
         self.kernel_manager.client_factory = self.kernel_client_class
         self.kernel_manager.start_kernel(extra_arguments=self.kernel_argv)
         atexit.register(self.kernel_manager.cleanup_ipc_files)

IPython/kernel/launcher.py

@@ -1,21 +1,8 b''
 """Utilities for launching kernels
-
-Authors:
-
-* Min Ragan-Kelley
-
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2013  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
@@ -24,9 +11,6 b' from subprocess import Popen, PIPE'
 from IPython.utils.encoding import getdefaultencoding
 from IPython.utils.py3compat import cast_bytes_py2
 
-#-----------------------------------------------------------------------------
-# Launching Kernels
-#-----------------------------------------------------------------------------
 
 def swallow_argv(argv, aliases=None, flags=None):
     """strip frontend-specific aliases and flags from an argument list
@@ -136,7 +120,7 b' def make_ipkernel_cmd(code, executable=None, extra_arguments=[], **kw):'
     return arguments
 
 
-def launch_kernel(cmd, stdin=None, stdout=None, stderr=None,
+def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None,
                         independent=False,
                         cwd=None, ipython_kernel=True,
                         **kw
@@ -221,7 +205,7 b' def launch_kernel(cmd, stdin=None, stdout=None, stderr=None,'
         if independent:
             proc = Popen(cmd,
                          creationflags=512, # CREATE_NEW_PROCESS_GROUP
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, env=env)
         else:
             if ipython_kernel:
                 try:
@@ -238,7 +222,7 b' def launch_kernel(cmd, stdin=None, stdout=None, stderr=None,'
             
             
             proc = Popen(cmd,
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
 
         # Attach the interrupt event to the Popen objet so it can be used later.
         proc.win32_interrupt_event = interrupt_event
@@ -246,12 +230,12 b' def launch_kernel(cmd, stdin=None, stdout=None, stderr=None,'
     else:
         if independent:
             proc = Popen(cmd, preexec_fn=lambda: os.setsid(),
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
         else:
             if ipython_kernel:
                 cmd += ['--parent=1']
             proc = Popen(cmd,
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
 
     # Clean up pipes created to work around Popen bug.
     if redirect_in:

IPython/kernel/manager.py

@@ -1,23 +1,17 b''
 """Base class to manage a running kernel"""
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2013  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 from __future__ import absolute_import
 
 # Standard library imports
+import os
 import re
 import signal
 import sys
 import time
+import warnings
 
 import zmq
 
@@ -25,12 +19,14 b' import zmq'
 from IPython.config.configurable import LoggingConfigurable
 from IPython.utils.importstring import import_item
 from IPython.utils.localinterfaces import is_local_ip, local_ips
+from IPython.utils.path import get_ipython_dir
 from IPython.utils.traitlets import (
     Any, Instance, Unicode, List, Bool, Type, DottedObjectName
 )
 from IPython.kernel import (
     make_ipkernel_cmd,
     launch_kernel,
+    kernelspec,
 )
 from .connect import ConnectionFileMixin
 from .zmq.session import Session
@@ -38,9 +34,6 b' from .managerabc import ('
     KernelManagerABC
 )
 
-#-----------------------------------------------------------------------------
-# Main kernel manager class
-#-----------------------------------------------------------------------------
 
 class KernelManager(LoggingConfigurable, ConnectionFileMixin):
     """Manages a single kernel in a subprocess on this host.
@@ -67,9 +60,27 b' class KernelManager(LoggingConfigurable, ConnectionFileMixin):'
     # The kernel process with which the KernelManager is communicating.
     # generally a Popen instance
     kernel = Any()
+    
+    kernel_spec_manager = Instance(kernelspec.KernelSpecManager)
+    
+    def _kernel_spec_manager_default(self):
+        return kernelspec.KernelSpecManager(ipython_dir=self.ipython_dir)
+    
+    kernel_name = Unicode('python')
+    
+    kernel_spec = Instance(kernelspec.KernelSpec)
+    
+    def _kernel_spec_default(self):
+        return self.kernel_spec_manager.get_kernel_spec(self.kernel_name)
+    
+    def _kernel_name_changed(self, name, old, new):
+        self.kernel_spec = self.kernel_spec_manager.get_kernel_spec(new)
+        self.ipython_kernel = new in {'python', 'python2', 'python3'}
 
     kernel_cmd = List(Unicode, config=True,
-        help="""The Popen Command to launch the kernel.
+        help="""DEPRECATED: Use kernel_name instead.
+        
+        The Popen Command to launch the kernel.
         Override this if you have a custom kernel.
         If kernel_cmd is specified in a configuration file,
         IPython does not pass any arguments to the kernel,
@@ -81,9 +92,15 b' class KernelManager(LoggingConfigurable, ConnectionFileMixin):'
     )
 
     def _kernel_cmd_changed(self, name, old, new):
+        warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to "
+                      "start different kernels.")
         self.ipython_kernel = False
 
     ipython_kernel = Bool(True)
+    
+    ipython_dir = Unicode()
+    def _ipython_dir_default(self):
+        return get_ipython_dir()
 
     # Protected traits
     _launch_args = Any()
@@ -150,11 +167,15 b' class KernelManager(LoggingConfigurable, ConnectionFileMixin):'
         """replace templated args (e.g. {connection_file})"""
         if self.kernel_cmd:
             cmd = self.kernel_cmd
-        else:
+        elif self.kernel_name == 'python':
+            # The native kernel gets special handling
             cmd = make_ipkernel_cmd(
                 'from IPython.kernel.zmq.kernelapp import main; main()',
                 **kw
             )
+        else:
+            cmd = self.kernel_spec.argv
+
         ns = dict(connection_file=self.connection_file)
         ns.update(self._launch_args)
         
@@ -211,8 +232,15 b' class KernelManager(LoggingConfigurable, ConnectionFileMixin):'
         self._launch_args = kw.copy()
         # build the Popen cmd
         kernel_cmd = self.format_kernel_cmd(**kw)
+        if self.kernel_cmd:
+            # If kernel_cmd has been set manually, don't refer to a kernel spec
+            env = os.environ
+        else:
+            # Environment variables from kernel spec are added to os.environ
+            env = os.environ.copy()
+            env.update(self.kernel_spec.env or {})
         # launch the kernel subprocess
-        self.kernel = self._launch_kernel(kernel_cmd,
+        self.kernel = self._launch_kernel(kernel_cmd, env=env,
                                     ipython_kernel=self.ipython_kernel,
                                     **kw)
         self.start_restarter()
@@ -381,9 +409,5 b' class KernelManager(LoggingConfigurable, ConnectionFileMixin):'
             return False
 
 
-#-----------------------------------------------------------------------------
-# ABC Registration
-#-----------------------------------------------------------------------------
-
 KernelManagerABC.register(KernelManager)