Merge pull request #5598 from takluyver/kernelspec

Kernel registry (IPEP 25)

IPython/kernel/kernelspec.py

@@ -0,0 +1,142
+import io
+import json
+import os
+import sys
+
+pjoin = os.path.join
+
+from IPython.utils.path import get_ipython_dir
+from IPython.utils.py3compat import PY3
+from IPython.utils.traitlets import HasTraits, List, Unicode, Dict
+
+if os.name == 'nt':
+    programdata = os.environ.get('PROGRAMDATA', None)
+    if programdata:
+        SYSTEM_KERNEL_DIR = pjoin(programdata, 'ipython', 'kernels')
+    else:  # PROGRAMDATA is not defined by default on XP.
+        SYSTEM_KERNEL_DIR = None
+else:
+    SYSTEM_KERNEL_DIR = "/usr/share/ipython/kernels"
+    
+NATIVE_KERNEL_NAME = 'python3' if PY3 else 'python2'
+
+class KernelSpec(HasTraits):
+    argv = List()
+    display_name = Unicode()
+    language = Unicode()
+    codemirror_mode = None
+    env = Dict()
+    
+    resource_dir = Unicode()
+    
+    def __init__(self, resource_dir, argv, display_name, language,
+                 codemirror_mode=None):
+        super(KernelSpec, self).__init__(resource_dir=resource_dir, argv=argv,
+                display_name=display_name, language=language,
+                codemirror_mode=codemirror_mode)
+        if not self.codemirror_mode:
+            self.codemirror_mode = self.language
+    
+    @classmethod
+    def from_resource_dir(cls, resource_dir):
+        """Create a KernelSpec object by reading kernel.json
+        
+        Pass the path to the *directory* containing kernel.json.
+        """
+        kernel_file = pjoin(resource_dir, 'kernel.json')
+        with io.open(kernel_file, 'r', encoding='utf-8') as f:
+            kernel_dict = json.load(f)
+        return cls(resource_dir=resource_dir, **kernel_dict)
+
+def _is_kernel_dir(path):
+    """Is ``path`` a kernel directory?"""
+    return os.path.isdir(path) and os.path.isfile(pjoin(path, 'kernel.json'))
+
+def _list_kernels_in(dir):
+    """Return a mapping of kernel names to resource directories from dir.
+    
+    If dir is None or does not exist, returns an empty dict.
+    """
+    if dir is None or not os.path.isdir(dir):
+        return {}
+    return {f.lower(): pjoin(dir, f) for f in os.listdir(dir)
+                        if _is_kernel_dir(pjoin(dir, f))}
+
+class NoSuchKernel(KeyError):
+    def __init__(self, name):
+        self.name = name
+
+class KernelSpecManager(HasTraits):
+    ipython_dir = Unicode()
+    def _ipython_dir_default(self):
+        return get_ipython_dir()
+
+    user_kernel_dir = Unicode()
+    def _user_kernel_dir_default(self):
+        return pjoin(self.ipython_dir, 'kernels')
+    
+    kernel_dirs = List(
+        help="List of kernel directories to search. Later ones take priority over earlier."    
+    )    
+    def _kernel_dirs_default(self):
+        return [
+            SYSTEM_KERNEL_DIR,
+            self.user_kernel_dir,
+        ]
+
+    def _make_native_kernel_dir(self):
+        """Makes a kernel directory for the native kernel.
+        
+        The native kernel is the kernel using the same Python runtime as this
+        process. This will put its informatino in the user kernels directory.
+        """
+        path = pjoin(self.user_kernel_dir, NATIVE_KERNEL_NAME)
+        os.makedirs(path, mode=0o755)
+        with open(pjoin(path, 'kernel.json'), 'w') as f:
+            json.dump({'argv':[NATIVE_KERNEL_NAME, '-c',
+                               'from IPython.kernel.zmq.kernelapp import main; main()',
+                                '-f', '{connection_file}'],
+                       'display_name': 'Python 3' if PY3 else 'Python 2',
+                       'language': 'python',
+                       'codemirror_mode': {'name': 'python',
+                                           'version': sys.version_info[0]},
+                      },
+                      f, indent=1)
+        # TODO: Copy icons into directory
+        return path
+    
+    def find_kernel_specs(self):
+        """Returns a dict mapping kernel names to resource directories."""
+        d = {}
+        for kernel_dir in self.kernel_dirs:
+            d.update(_list_kernels_in(kernel_dir))
+        
+        if NATIVE_KERNEL_NAME not in d:
+            d[NATIVE_KERNEL_NAME] = self._make_native_kernel_dir()
+        return d
+        # TODO: Caching?
+    
+    def get_kernel_spec(self, kernel_name):
+        """Returns a :class:`KernelSpec` instance for the given kernel_name.
+        
+        Raises :exc:`NoSuchKernel` if the given kernel name is not found.
+        """
+        if kernel_name == 'python':
+            kernel_name = NATIVE_KERNEL_NAME
+        d = self.find_kernel_specs()
+        try:
+            resource_dir = d[kernel_name.lower()]
+        except KeyError:
+            raise NoSuchKernel(kernel_name)
+        return KernelSpec.from_resource_dir(resource_dir)
+
+def find_kernel_specs():
+    """Returns a dict mapping kernel names to resource directories."""
+    return KernelSpecManager().find_kernel_specs()
+
+def get_kernel_spec(kernel_name):
+    """Returns a :class:`KernelSpec` instance for the given kernel_name.
+    
+    Raises KeyError if the given kernel name is not found.
+    """
+    return KernelSpecManager().get_kernel_spec(kernel_name)
  No newline at end of file

IPython/kernel/tests/test_kernelspec.py

@@ -0,0 +1,39
+import json
+import os
+from os.path import join as pjoin
+import unittest
+
+from IPython.utils.tempdir import TemporaryDirectory
+from IPython.kernel import kernelspec
+
+sample_kernel_json = {'argv':['cat', '{connection_file}'],
+                      'display_name':'Test kernel',
+                      'language':'bash',
+                     }
+
+class KernelSpecTests(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = td = TemporaryDirectory()
+        self.sample_kernel_dir = pjoin(td.name, 'kernels', 'Sample')
+        os.makedirs(self.sample_kernel_dir)
+        json_file = pjoin(self.sample_kernel_dir, 'kernel.json')
+        with open(json_file, 'w') as f:
+            json.dump(sample_kernel_json, f)
+
+        self.ksm = kernelspec.KernelSpecManager(ipython_dir=td.name)
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+    def test_find_kernel_specs(self):
+        kernels = self.ksm.find_kernel_specs()
+        self.assertEqual(kernels['sample'], self.sample_kernel_dir)
+
+    def test_get_kernel_spec(self):
+        ks = self.ksm.get_kernel_spec('SAMPLE')  # Case insensitive
+        self.assertEqual(ks.resource_dir, self.sample_kernel_dir)
+        self.assertEqual(ks.argv, sample_kernel_json['argv'])
+        self.assertEqual(ks.display_name, sample_kernel_json['display_name'])
+        self.assertEqual(ks.language, sample_kernel_json['language'])
+        self.assertEqual(ks.codemirror_mode, sample_kernel_json['language'])
+        self.assertEqual(ks.env, {})
  No newline at end of file

IPython/consoleapp.py

@@ -1,387 +1,399
 """ A minimal application base mixin for all ZMQ based IPython frontends.
 
 This is not a complete console app, as subprocess will not be able to receive
 input, there is no real readline support, among other limitations. This is a
 refactoring of what used to be the IPython/qt/console/qtconsoleapp.py
 
 Authors:
 
 * Evan Patterson
 * Min RK
 * Erik Tollerud
 * Fernando Perez
 * Bussonnier Matthias
 * Thomas Kluyver
 * Paul Ivanov
 
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # stdlib imports
 import atexit
 import json
 import os
 import signal
 import sys
 import uuid
 
 
 # Local imports
 from IPython.config.application import boolean_flag
 from IPython.core.profiledir import ProfileDir
 from IPython.kernel.blocking import BlockingKernelClient
 from IPython.kernel import KernelManager
 from IPython.kernel import tunnel_to_kernel, find_connection_file, swallow_argv
+from IPython.kernel.kernelspec import NoSuchKernel
 from IPython.utils.path import filefind
 from IPython.utils.py3compat import str_to_bytes
 from IPython.utils.traitlets import (
     Dict, List, Unicode, CUnicode, Int, CBool, Any
 )
 from IPython.kernel.zmq.kernelapp import (
     kernel_flags,
     kernel_aliases,
     IPKernelApp
 )
 from IPython.kernel.zmq.pylab.config import InlineBackend
 from IPython.kernel.zmq.session import Session, default_secure
 from IPython.kernel.zmq.zmqshell import ZMQInteractiveShell
 from IPython.kernel.connect import ConnectionFileMixin
 
 #-----------------------------------------------------------------------------
 # Network Constants
 #-----------------------------------------------------------------------------
 
 from IPython.utils.localinterfaces import localhost
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = dict(kernel_flags)
 
 # the flags that are specific to the frontend
 # these must be scrubbed before being passed to the kernel,
 # or it will raise an error on unrecognized flags
 app_flags = {
     'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}},
             "Connect to an existing kernel. If no argument specified, guess most recent"),
 }
 app_flags.update(boolean_flag(
     'confirm-exit', 'IPythonConsoleApp.confirm_exit',
     """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
        to force a direct exit without any confirmation.
     """,
     """Don't prompt the user when exiting. This will terminate the kernel
        if it is owned by the frontend, and leave it alive if it is external.
     """
 ))
 flags.update(app_flags)
 
 aliases = dict(kernel_aliases)
 
 # also scrub aliases from the frontend
 app_aliases = dict(
     ip = 'IPythonConsoleApp.ip',
     transport = 'IPythonConsoleApp.transport',
     hb = 'IPythonConsoleApp.hb_port',
     shell = 'IPythonConsoleApp.shell_port',
     iopub = 'IPythonConsoleApp.iopub_port',
     stdin = 'IPythonConsoleApp.stdin_port',
     existing = 'IPythonConsoleApp.existing',
     f = 'IPythonConsoleApp.connection_file',
 
+    kernel = 'IPythonConsoleApp.kernel_name',
 
     ssh = 'IPythonConsoleApp.sshserver',
 )
 aliases.update(app_aliases)
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # IPythonConsole
 #-----------------------------------------------------------------------------
 
 classes = [IPKernelApp, ZMQInteractiveShell, KernelManager, ProfileDir, Session, InlineBackend]
 
 class IPythonConsoleApp(ConnectionFileMixin):
     name = 'ipython-console-mixin'
 
     description = """
         The IPython Mixin Console.
         
         This class contains the common portions of console client (QtConsole,
         ZMQ-based terminal console, etc).  It is not a full console, in that
         launched terminal subprocesses will not be able to accept input.
         
         The Console using this mixing supports various extra features beyond
         the single-process Terminal IPython shell, such as connecting to
         existing kernel, via:
         
             ipython <appname> --existing
         
         as well as tunnel via SSH
         
     """
 
     classes = classes
     flags = Dict(flags)
     aliases = Dict(aliases)
     kernel_manager_class = KernelManager
     kernel_client_class = BlockingKernelClient
 
     kernel_argv = List(Unicode)
     # frontend flags&aliases to be stripped when building kernel_argv
     frontend_flags = Any(app_flags)
     frontend_aliases = Any(app_aliases)
 
     # create requested profiles by default, if they don't exist:
     auto_create = CBool(True)
     # connection info:
     
     sshserver = Unicode('', config=True,
         help="""The SSH server to use to connect to the kernel.""")
     sshkey = Unicode('', config=True,
         help="""Path to the ssh key to use for logging in to the ssh server.""")
     
     hb_port = Int(0, config=True,
         help="set the heartbeat port [default: random]")
     shell_port = Int(0, config=True,
         help="set the shell (ROUTER) port [default: random]")
     iopub_port = Int(0, config=True,
         help="set the iopub (PUB) port [default: random]")
     stdin_port = Int(0, config=True,
         help="set the stdin (DEALER) port [default: random]")
     connection_file = Unicode('', config=True,
         help="""JSON file in which to store connection info [default: kernel-<pid>.json]
 
         This file will contain the IP, ports, and authentication key needed to connect
         clients to this kernel. By default, this file will be created in the security-dir
         of the current profile, but can be specified by absolute path.
         """)
     def _connection_file_default(self):
         return 'kernel-%i.json' % os.getpid()
 
     existing = CUnicode('', config=True,
         help="""Connect to an already running kernel""")
 
+    kernel_name = Unicode('python', config=True,
+        help="""The name of the default kernel to start.""")
+
     confirm_exit = CBool(True, config=True,
         help="""
         Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
         to force a direct exit without any confirmation.""",
     )
 
 
     def build_kernel_argv(self, argv=None):
         """build argv to be passed to kernel subprocess"""
         if argv is None:
             argv = sys.argv[1:]
         self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags)
         # kernel should inherit default config file from frontend
         self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name)
     
     def init_connection_file(self):
         """find the connection file, and load the info if found.
         
         The current working directory and the current profile's security
         directory will be searched for the file if it is not given by
         absolute path.
         
         When attempting to connect to an existing kernel and the `--existing`
         argument does not match an existing file, it will be interpreted as a
         fileglob, and the matching file in the current profile's security dir
         with the latest access time will be used.
         
         After this method is called, self.connection_file contains the *full path*
         to the connection file, never just its name.
         """
         if self.existing:
             try:
                 cf = find_connection_file(self.existing)
             except Exception:
                 self.log.critical("Could not find existing kernel connection file %s", self.existing)
                 self.exit(1)
             self.log.debug("Connecting to existing kernel: %s" % cf)
             self.connection_file = cf
         else:
             # not existing, check if we are going to write the file
             # and ensure that self.connection_file is a full path, not just the shortname
             try:
                 cf = find_connection_file(self.connection_file)
             except Exception:
                 # file might not exist
                 if self.connection_file == os.path.basename(self.connection_file):
                     # just shortname, put it in security dir
                     cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
                 else:
                     cf = self.connection_file
                 self.connection_file = cf
         
         # should load_connection_file only be used for existing?
         # as it is now, this allows reusing ports if an existing
         # file is requested
         try:
             self.load_connection_file()
         except Exception:
             self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
             self.exit(1)
     
     def load_connection_file(self):
         """load ip/port/hmac config from JSON connection file"""
         # this is identical to IPKernelApp.load_connection_file
         # perhaps it can be centralized somewhere?
         try:
             fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
         except IOError:
             self.log.debug("Connection File not found: %s", self.connection_file)
             return
         self.log.debug(u"Loading connection file %s", fname)
         with open(fname) as f:
             cfg = json.load(f)
         self.transport = cfg.get('transport', 'tcp')
         self.ip = cfg.get('ip', localhost())
         
         for channel in ('hb', 'shell', 'iopub', 'stdin', 'control'):
             name = channel + '_port'
             if getattr(self, name) == 0 and name in cfg:
                 # not overridden by config or cl_args
                 setattr(self, name, cfg[name])
         if 'key' in cfg:
             self.config.Session.key = str_to_bytes(cfg['key'])
         if 'signature_scheme' in cfg:
             self.config.Session.signature_scheme = cfg['signature_scheme']
     
     def init_ssh(self):
         """set up ssh tunnels, if needed."""
         if not self.existing or (not self.sshserver and not self.sshkey):
             return
         self.load_connection_file()
         
         transport = self.transport
         ip = self.ip
         
         if transport != 'tcp':
             self.log.error("Can only use ssh tunnels with TCP sockets, not %s", transport)
             sys.exit(-1)
         
         if self.sshkey and not self.sshserver:
             # specifying just the key implies that we are connecting directly
             self.sshserver = ip
             ip = localhost()
         
         # build connection dict for tunnels:
         info = dict(ip=ip,
                     shell_port=self.shell_port,
                     iopub_port=self.iopub_port,
                     stdin_port=self.stdin_port,
                     hb_port=self.hb_port
         )
         
         self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver))
         
         # tunnels return a new set of ports, which will be on localhost:
         self.ip = localhost()
         try:
             newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
         except:
             # even catch KeyboardInterrupt
             self.log.error("Could not setup tunnels", exc_info=True)
             self.exit(1)
         
         self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
         
         cf = self.connection_file
         base,ext = os.path.splitext(cf)
         base = os.path.basename(base)
         self.connection_file = os.path.basename(base)+'-ssh'+ext
         self.log.info("To connect another client via this tunnel, use:")
         self.log.info("--existing %s" % self.connection_file)
     
     def _new_connection_file(self):
         cf = ''
         while not cf:
             # we don't need a 128b id to distinguish kernels, use more readable
             # 48b node segment (12 hex chars).  Users running more than 32k simultaneous
             # kernels can subclass.
             ident = str(uuid.uuid4()).split('-')[-1]
             cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident)
             # only keep if it's actually new.  Protect against unlikely collision
             # in 48b random search space
             cf = cf if not os.path.exists(cf) else ''
         return cf
 
     def init_kernel_manager(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.
         if self.existing:
             self.kernel_manager = None
             return
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
         # Create a KernelManager and start a kernel.
-        self.kernel_manager = self.kernel_manager_class(
-                                ip=self.ip,
-                                transport=self.transport,
-                                shell_port=self.shell_port,
-                                iopub_port=self.iopub_port,
-                                stdin_port=self.stdin_port,
-                                hb_port=self.hb_port,
-                                connection_file=self.connection_file,
-                                parent=self,
-        )
+        try:
+            self.kernel_manager = self.kernel_manager_class(
+                                    ip=self.ip,
+                                    transport=self.transport,
+                                    shell_port=self.shell_port,
+                                    iopub_port=self.iopub_port,
+                                    stdin_port=self.stdin_port,
+                                    hb_port=self.hb_port,
+                                    connection_file=self.connection_file,
+                                    kernel_name=self.kernel_name,
+                                    parent=self,
+                                    ipython_dir=self.ipython_dir,
+            )
+        except NoSuchKernel:
+            self.log.critical("Could not find kernel %s", self.kernel_name)
+            self.exit(1)
+
         self.kernel_manager.client_factory = self.kernel_client_class
         self.kernel_manager.start_kernel(extra_arguments=self.kernel_argv)
         atexit.register(self.kernel_manager.cleanup_ipc_files)
 
         if self.sshserver:
             # ssh, write new connection file
             self.kernel_manager.write_connection_file()
 
         # in case KM defaults / ssh writing changes things:
         km = self.kernel_manager
         self.shell_port=km.shell_port
         self.iopub_port=km.iopub_port
         self.stdin_port=km.stdin_port
         self.hb_port=km.hb_port
         self.connection_file = km.connection_file
 
         atexit.register(self.kernel_manager.cleanup_connection_file)
 
     def init_kernel_client(self):
         if self.kernel_manager is not None:
             self.kernel_client = self.kernel_manager.client()
         else:
             self.kernel_client = self.kernel_client_class(
                                 ip=self.ip,
                                 transport=self.transport,
                                 shell_port=self.shell_port,
                                 iopub_port=self.iopub_port,
                                 stdin_port=self.stdin_port,
                                 hb_port=self.hb_port,
                                 connection_file=self.connection_file,
                                 parent=self,
             )
 
         self.kernel_client.start_channels()
 
 
 
     def initialize(self, argv=None):
         """
         Classes which mix this class in should call:
                IPythonConsoleApp.initialize(self,argv)
         """
         self.init_connection_file()
         default_secure(self.config)
         self.init_ssh()
         self.init_kernel_manager()
         self.init_kernel_client()
 

IPython/kernel/launcher.py

@@ -1,272 +1,256
 """Utilities for launching kernels
-
-Authors:
-
-* Min Ragan-Kelley
-
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2013  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 from subprocess import Popen, PIPE
 
 from IPython.utils.encoding import getdefaultencoding
 from IPython.utils.py3compat import cast_bytes_py2
 
-#-----------------------------------------------------------------------------
-# Launching Kernels
-#-----------------------------------------------------------------------------
 
 def swallow_argv(argv, aliases=None, flags=None):
     """strip frontend-specific aliases and flags from an argument list
     
     For use primarily in frontend apps that want to pass a subset of command-line
     arguments through to a subprocess, where frontend-specific flags and aliases
     should be removed from the list.
     
     Parameters
     ----------
     
     argv : list(str)
         The starting argv, to be filtered
     aliases : container of aliases (dict, list, set, etc.)
         The frontend-specific aliases to be removed
     flags : container of flags (dict, list, set, etc.)
         The frontend-specific flags to be removed
     
     Returns
     -------
     
     argv : list(str)
         The argv list, excluding flags and aliases that have been stripped
     """
     
     if aliases is None:
         aliases = set()
     if flags is None:
         flags = set()
     
     stripped = list(argv) # copy
     
     swallow_next = False
     was_flag = False
     for a in argv:
         if a == '--':
             break
         if swallow_next:
             swallow_next = False
             # last arg was an alias, remove the next one
             # *unless* the last alias has a no-arg flag version, in which
             # case, don't swallow the next arg if it's also a flag:
             if not (was_flag and a.startswith('-')):
                 stripped.remove(a)
                 continue
         if a.startswith('-'):
             split = a.lstrip('-').split('=')
             name = split[0]
             # we use startswith because argparse accepts any arg to be specified
             # by any leading section, as long as it is unique,
             # so `--no-br` means `--no-browser` in the notebook, etc.
             if any(alias.startswith(name) for alias in aliases):
                 stripped.remove(a)
                 if len(split) == 1:
                     # alias passed with arg via space
                     swallow_next = True
                     # could have been a flag that matches an alias, e.g. `existing`
                     # in which case, we might not swallow the next arg
                     was_flag = name in flags
             elif len(split) == 1 and any(flag.startswith(name) for flag in flags):
                 # strip flag, but don't swallow next, as flags don't take args
                 stripped.remove(a)
     
     # return shortened list
     return stripped
 
 
 def make_ipkernel_cmd(code, executable=None, extra_arguments=[], **kw):
     """Build Popen command list for launching an IPython kernel.
 
     Parameters
     ----------
     code : str,
         A string of Python code that imports and executes a kernel entry point.
 
     executable : str, optional (default sys.executable)
         The Python executable to use for the kernel process.
 
     extra_arguments : list, optional
         A list of extra arguments to pass when executing the launch code.
     
     Returns
     -------
     
     A Popen command list
     """
     
     # Build the kernel launch command.
     if executable is None:
         executable = sys.executable
     arguments = [ executable, '-c', code, '-f', '{connection_file}' ]
     arguments.extend(extra_arguments)
 
     # Spawn a kernel.
     if sys.platform == 'win32':
 
         # If the kernel is running on pythonw and stdout/stderr are not been
         # re-directed, it will crash when more than 4KB of data is written to
         # stdout or stderr. This is a bug that has been with Python for a very
         # long time; see http://bugs.python.org/issue706263.
         # A cleaner solution to this problem would be to pass os.devnull to
         # Popen directly. Unfortunately, that does not work.
         if executable.endswith('pythonw.exe'):
             arguments.append('--no-stdout')
             arguments.append('--no-stderr')
 
     return arguments
 
 
-def launch_kernel(cmd, stdin=None, stdout=None, stderr=None,
+def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None,
                         independent=False,
                         cwd=None, ipython_kernel=True,
                         **kw
                         ):
     """ Launches a localhost kernel, binding to the specified ports.
 
     Parameters
     ----------
     cmd : Popen list,
         A string of Python code that imports and executes a kernel entry point.
 
     stdin, stdout, stderr : optional (default None)
         Standards streams, as defined in subprocess.Popen.
 
     independent : bool, optional (default False)
         If set, the kernel process is guaranteed to survive if this process
         dies. If not set, an effort is made to ensure that the kernel is killed
         when this process dies. Note that in this case it is still good practice
         to kill kernels manually before exiting.
 
     cwd : path, optional
         The working dir of the kernel process (default: cwd of this process).
 
     ipython_kernel : bool, optional
         Whether the kernel is an official IPython one,
         and should get a bit of special treatment.
 
     Returns
     -------
     
     Popen instance for the kernel subprocess
     """
 
     # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
     # are invalid. Unfortunately, there is in general no way to detect whether
     # they are valid.  The following two blocks redirect them to (temporary)
     # pipes in certain important cases.
 
     # If this process has been backgrounded, our stdin is invalid. Since there
     # is no compelling reason for the kernel to inherit our stdin anyway, we'll
     # place this one safe and always redirect.
     redirect_in = True
     _stdin = PIPE if stdin is None else stdin
 
     # If this process in running on pythonw, we know that stdin, stdout, and
     # stderr are all invalid.
     redirect_out = sys.executable.endswith('pythonw.exe')
     if redirect_out:
         _stdout = PIPE if stdout is None else stdout
         _stderr = PIPE if stderr is None else stderr
     else:
         _stdout, _stderr = stdout, stderr
     
     encoding = getdefaultencoding(prefer_stream=False)
     
     # Spawn a kernel.
     if sys.platform == 'win32':
         # Popen on Python 2 on Windows cannot handle unicode args or cwd
         cmd = [ cast_bytes_py2(c, encoding) for c in cmd ]
         if cwd:
             cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii')
         
         from IPython.kernel.zmq.parentpoller import ParentPollerWindows
         # Create a Win32 event for interrupting the kernel.
         interrupt_event = ParentPollerWindows.create_interrupt_event()
         if ipython_kernel:
             cmd += [ '--interrupt=%i' % interrupt_event ]
 
             # If the kernel is running on pythonw and stdout/stderr are not been
             # re-directed, it will crash when more than 4KB of data is written to
             # stdout or stderr. This is a bug that has been with Python for a very
             # long time; see http://bugs.python.org/issue706263.
             # A cleaner solution to this problem would be to pass os.devnull to
             # Popen directly. Unfortunately, that does not work.
             if cmd[0].endswith('pythonw.exe'):
                 if stdout is None:
                     cmd.append('--no-stdout')
                 if stderr is None:
                     cmd.append('--no-stderr')
 
         # Launch the kernel process.
         if independent:
             proc = Popen(cmd,
                          creationflags=512, # CREATE_NEW_PROCESS_GROUP
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, env=env)
         else:
             if ipython_kernel:
                 try:
                     from _winapi import DuplicateHandle, GetCurrentProcess, \
                         DUPLICATE_SAME_ACCESS
                 except:
                     from _subprocess import DuplicateHandle, GetCurrentProcess, \
                         DUPLICATE_SAME_ACCESS
                 pid = GetCurrentProcess()
                 handle = DuplicateHandle(pid, pid, pid, 0,
                                          True, # Inheritable by new processes.
                                          DUPLICATE_SAME_ACCESS)
                 cmd +=[ '--parent=%i' % handle ]
             
             
             proc = Popen(cmd,
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
 
         # Attach the interrupt event to the Popen objet so it can be used later.
         proc.win32_interrupt_event = interrupt_event
 
     else:
         if independent:
             proc = Popen(cmd, preexec_fn=lambda: os.setsid(),
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
         else:
             if ipython_kernel:
                 cmd += ['--parent=1']
             proc = Popen(cmd,
-                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=os.environ)
+                         stdin=_stdin, stdout=_stdout, stderr=_stderr, cwd=cwd, env=env)
 
     # Clean up pipes created to work around Popen bug.
     if redirect_in:
         if stdin is None:
             proc.stdin.close()
     if redirect_out:
         if stdout is None:
             proc.stdout.close()
         if stderr is None:
             proc.stderr.close()
 
     return proc
 
 __all__ = [
     'swallow_argv',
     'make_ipkernel_cmd',
     'launch_kernel',
 ]

IPython/kernel/manager.py

@@ -1,389 +1,413
 """Base class to manage a running kernel"""
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2013  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 from __future__ import absolute_import
 
 # Standard library imports
+import os
 import re
 import signal
 import sys
 import time
+import warnings
 
 import zmq
 
 # Local imports
 from IPython.config.configurable import LoggingConfigurable
 from IPython.utils.importstring import import_item
 from IPython.utils.localinterfaces import is_local_ip, local_ips
+from IPython.utils.path import get_ipython_dir
 from IPython.utils.traitlets import (
     Any, Instance, Unicode, List, Bool, Type, DottedObjectName
 )
 from IPython.kernel import (
     make_ipkernel_cmd,
     launch_kernel,
+    kernelspec,
 )
 from .connect import ConnectionFileMixin
 from .zmq.session import Session
 from .managerabc import (
     KernelManagerABC
 )
 
-#-----------------------------------------------------------------------------
-# Main kernel manager class
-#-----------------------------------------------------------------------------
 
 class KernelManager(LoggingConfigurable, ConnectionFileMixin):
     """Manages a single kernel in a subprocess on this host.
 
     This version starts kernels with Popen.
     """
 
     # The PyZMQ Context to use for communication with the kernel.
     context = Instance(zmq.Context)
     def _context_default(self):
         return zmq.Context.instance()
 
     # The Session to use for communication with the kernel.
     session = Instance(Session)
     def _session_default(self):
         return Session(parent=self)
 
     # the class to create with our `client` method
     client_class = DottedObjectName('IPython.kernel.blocking.BlockingKernelClient')
     client_factory = Type()
     def _client_class_changed(self, name, old, new):
         self.client_factory = import_item(str(new))
 
     # The kernel process with which the KernelManager is communicating.
     # generally a Popen instance
     kernel = Any()
+    
+    kernel_spec_manager = Instance(kernelspec.KernelSpecManager)
+    
+    def _kernel_spec_manager_default(self):
+        return kernelspec.KernelSpecManager(ipython_dir=self.ipython_dir)
+    
+    kernel_name = Unicode('python')
+    
+    kernel_spec = Instance(kernelspec.KernelSpec)
+    
+    def _kernel_spec_default(self):
+        return self.kernel_spec_manager.get_kernel_spec(self.kernel_name)
+    
+    def _kernel_name_changed(self, name, old, new):
+        self.kernel_spec = self.kernel_spec_manager.get_kernel_spec(new)
+        self.ipython_kernel = new in {'python', 'python2', 'python3'}
 
     kernel_cmd = List(Unicode, config=True,
-        help="""The Popen Command to launch the kernel.
+        help="""DEPRECATED: Use kernel_name instead.
+        
+        The Popen Command to launch the kernel.
         Override this if you have a custom kernel.
         If kernel_cmd is specified in a configuration file,
         IPython does not pass any arguments to the kernel,
         because it cannot make any assumptions about the 
         arguments that the kernel understands. In particular,
         this means that the kernel does not receive the
         option --debug if it given on the IPython command line.
         """
     )
 
     def _kernel_cmd_changed(self, name, old, new):
+        warnings.warn("Setting kernel_cmd is deprecated, use kernel_spec to "
+                      "start different kernels.")
         self.ipython_kernel = False
 
     ipython_kernel = Bool(True)
+    
+    ipython_dir = Unicode()
+    def _ipython_dir_default(self):
+        return get_ipython_dir()
 
     # Protected traits
     _launch_args = Any()
     _control_socket = Any()
 
     _restarter = Any()
 
     autorestart = Bool(False, config=True,
         help="""Should we autorestart the kernel if it dies."""
     )
 
     def __del__(self):
         self._close_control_socket()
         self.cleanup_connection_file()
 
     #--------------------------------------------------------------------------
     # Kernel restarter
     #--------------------------------------------------------------------------
 
     def start_restarter(self):
         pass
 
     def stop_restarter(self):
         pass
 
     def add_restart_callback(self, callback, event='restart'):
         """register a callback to be called when a kernel is restarted"""
         if self._restarter is None:
             return
         self._restarter.add_callback(callback, event)
 
     def remove_restart_callback(self, callback, event='restart'):
         """unregister a callback to be called when a kernel is restarted"""
         if self._restarter is None:
             return
         self._restarter.remove_callback(callback, event)
 
     #--------------------------------------------------------------------------
     # create a Client connected to our Kernel
     #--------------------------------------------------------------------------
 
     def client(self, **kwargs):
         """Create a client configured to connect to our kernel"""
         if self.client_factory is None:
             self.client_factory = import_item(self.client_class)
 
         kw = {}
         kw.update(self.get_connection_info())
         kw.update(dict(
             connection_file=self.connection_file,
             session=self.session,
             parent=self,
         ))
 
         # add kwargs last, for manual overrides
         kw.update(kwargs)
         return self.client_factory(**kw)
 
     #--------------------------------------------------------------------------
     # Kernel management
     #--------------------------------------------------------------------------
 
     def format_kernel_cmd(self, **kw):
         """replace templated args (e.g. {connection_file})"""
         if self.kernel_cmd:
             cmd = self.kernel_cmd
-        else:
+        elif self.kernel_name == 'python':
+            # The native kernel gets special handling
             cmd = make_ipkernel_cmd(
                 'from IPython.kernel.zmq.kernelapp import main; main()',
                 **kw
             )
+        else:
+            cmd = self.kernel_spec.argv
+
         ns = dict(connection_file=self.connection_file)
         ns.update(self._launch_args)
         
         pat = re.compile(r'\{([A-Za-z0-9_]+)\}')
         def from_ns(match):
             """Get the key out of ns if it's there, otherwise no change."""
             return ns.get(match.group(1), match.group())
         
         return [ pat.sub(from_ns, arg) for arg in cmd ]
 
     def _launch_kernel(self, kernel_cmd, **kw):
         """actually launch the kernel
 
         override in a subclass to launch kernel subprocesses differently
         """
         return launch_kernel(kernel_cmd, **kw)
 
     # Control socket used for polite kernel shutdown
 
     def _connect_control_socket(self):
         if self._control_socket is None:
             self._control_socket = self.connect_control()
             self._control_socket.linger = 100
 
     def _close_control_socket(self):
         if self._control_socket is None:
             return
         self._control_socket.close()
         self._control_socket = None
 
     def start_kernel(self, **kw):
         """Starts a kernel on this host in a separate process.
 
         If random ports (port=0) are being used, this method must be called
         before the channels are created.
 
         Parameters
         ----------
         **kw : optional
              keyword arguments that are passed down to build the kernel_cmd
              and launching the kernel (e.g. Popen kwargs).
         """
         if self.transport == 'tcp' and not is_local_ip(self.ip):
             raise RuntimeError("Can only launch a kernel on a local interface. "
                                "Make sure that the '*_address' attributes are "
                                "configured properly. "
                                "Currently valid addresses are: %s" % local_ips()
                                )
 
         # write connection file / get default ports
         self.write_connection_file()
 
         # save kwargs for use in restart
         self._launch_args = kw.copy()
         # build the Popen cmd
         kernel_cmd = self.format_kernel_cmd(**kw)
+        if self.kernel_cmd:
+            # If kernel_cmd has been set manually, don't refer to a kernel spec
+            env = os.environ
+        else:
+            # Environment variables from kernel spec are added to os.environ
+            env = os.environ.copy()
+            env.update(self.kernel_spec.env or {})
         # launch the kernel subprocess
-        self.kernel = self._launch_kernel(kernel_cmd,
+        self.kernel = self._launch_kernel(kernel_cmd, env=env,
                                     ipython_kernel=self.ipython_kernel,
                                     **kw)
         self.start_restarter()
         self._connect_control_socket()
 
     def _send_shutdown_request(self, restart=False):
         """TODO: send a shutdown request via control channel"""
         content = dict(restart=restart)
         msg = self.session.msg("shutdown_request", content=content)
         self.session.send(self._control_socket, msg)
 
     def shutdown_kernel(self, now=False, restart=False):
         """Attempts to the stop the kernel process cleanly.
 
         This attempts to shutdown the kernels cleanly by:
 
         1. Sending it a shutdown message over the shell channel.
         2. If that fails, the kernel is shutdown forcibly by sending it
            a signal.
 
         Parameters
         ----------
         now : bool
             Should the kernel be forcible killed *now*. This skips the
             first, nice shutdown attempt.
         restart: bool
             Will this kernel be restarted after it is shutdown. When this
             is True, connection files will not be cleaned up.
         """
         # Stop monitoring for restarting while we shutdown.
         self.stop_restarter()
 
         # FIXME: Shutdown does not work on Windows due to ZMQ errors!
         if now or sys.platform == 'win32':
             if self.has_kernel:
                 self._kill_kernel()
         else:
             # Don't send any additional kernel kill messages immediately, to give
             # the kernel a chance to properly execute shutdown actions. Wait for at
             # most 1s, checking every 0.1s.
             self._send_shutdown_request(restart=restart)
             for i in range(10):
                 if self.is_alive():
                     time.sleep(0.1)
                 else:
                     break
             else:
                 # OK, we've waited long enough.
                 if self.has_kernel:
                     self._kill_kernel()
 
         if not restart:
             self.cleanup_connection_file()
             self.cleanup_ipc_files()
         else:
             self.cleanup_ipc_files()
         
         self._close_control_socket()
 
     def restart_kernel(self, now=False, **kw):
         """Restarts a kernel with the arguments that were used to launch it.
 
         If the old kernel was launched with random ports, the same ports will be
         used for the new kernel. The same connection file is used again.
 
         Parameters
         ----------
         now : bool, optional
             If True, the kernel is forcefully restarted *immediately*, without
             having a chance to do any cleanup action.  Otherwise the kernel is
             given 1s to clean up before a forceful restart is issued.
 
             In all cases the kernel is restarted, the only difference is whether
             it is given a chance to perform a clean shutdown or not.
 
         **kw : optional
             Any options specified here will overwrite those used to launch the
             kernel.
         """
         if self._launch_args is None:
             raise RuntimeError("Cannot restart the kernel. "
                                "No previous call to 'start_kernel'.")
         else:
             # Stop currently running kernel.
             self.shutdown_kernel(now=now, restart=True)
 
             # Start new kernel.
             self._launch_args.update(kw)
             self.start_kernel(**self._launch_args)
 
             # FIXME: Messages get dropped in Windows due to probable ZMQ bug
             # unless there is some delay here.
             if sys.platform == 'win32':
                 time.sleep(0.2)
 
     @property
     def has_kernel(self):
         """Has a kernel been started that we are managing."""
         return self.kernel is not None
 
     def _kill_kernel(self):
         """Kill the running kernel.
 
         This is a private method, callers should use shutdown_kernel(now=True).
         """
         if self.has_kernel:
 
             # Signal the kernel to terminate (sends SIGKILL on Unix and calls
             # TerminateProcess() on Win32).
             try:
                 self.kernel.kill()
             except OSError as e:
                 # In Windows, we will get an Access Denied error if the process
                 # has already terminated. Ignore it.
                 if sys.platform == 'win32':
                     if e.winerror != 5:
                         raise
                 # On Unix, we may get an ESRCH error if the process has already
                 # terminated. Ignore it.
                 else:
                     from errno import ESRCH
                     if e.errno != ESRCH:
                         raise
 
             # Block until the kernel terminates.
             self.kernel.wait()
             self.kernel = None
         else:
             raise RuntimeError("Cannot kill kernel. No kernel is running!")
 
     def interrupt_kernel(self):
         """Interrupts the kernel by sending it a signal.
 
         Unlike ``signal_kernel``, this operation is well supported on all
         platforms.
         """
         if self.has_kernel:
             if sys.platform == 'win32':
                 from .zmq.parentpoller import ParentPollerWindows as Poller
                 Poller.send_interrupt(self.kernel.win32_interrupt_event)
             else:
                 self.kernel.send_signal(signal.SIGINT)
         else:
             raise RuntimeError("Cannot interrupt kernel. No kernel is running!")
 
     def signal_kernel(self, signum):
         """Sends a signal to the kernel.
 
         Note that since only SIGTERM is supported on Windows, this function is
         only useful on Unix systems.
         """
         if self.has_kernel:
             self.kernel.send_signal(signum)
         else:
             raise RuntimeError("Cannot signal kernel. No kernel is running!")
 
     def is_alive(self):
         """Is the kernel process still running?"""
         if self.has_kernel:
             if self.kernel.poll() is None:
                 return True
             else:
                 return False
         else:
             # we don't have a kernel
             return False
 
 
-#-----------------------------------------------------------------------------
-# ABC Registration
-#-----------------------------------------------------------------------------
-
 KernelManagerABC.register(KernelManager)