finish kernel shims

ipython_kernel/launcher.py

@@ -0,0 +1,226 b''
+"""Utilities for launching kernels
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import os
+import sys
+from subprocess import Popen, PIPE
+
+from IPython.utils.encoding import getdefaultencoding
+from IPython.utils.py3compat import cast_bytes_py2
+
+
+def swallow_argv(argv, aliases=None, flags=None):
+    """strip frontend-specific aliases and flags from an argument list
+
+    For use primarily in frontend apps that want to pass a subset of command-line
+    arguments through to a subprocess, where frontend-specific flags and aliases
+    should be removed from the list.
+
+    Parameters
+    ----------
+
+    argv : list(str)
+        The starting argv, to be filtered
+    aliases : container of aliases (dict, list, set, etc.)
+        The frontend-specific aliases to be removed
+    flags : container of flags (dict, list, set, etc.)
+        The frontend-specific flags to be removed
+
+    Returns
+    -------
+
+    argv : list(str)
+        The argv list, excluding flags and aliases that have been stripped
+    """
+
+    if aliases is None:
+        aliases = set()
+    if flags is None:
+        flags = set()
+
+    stripped = list(argv) # copy
+
+    swallow_next = False
+    was_flag = False
+    for a in argv:
+        if a == '--':
+            break
+        if swallow_next:
+            swallow_next = False
+            # last arg was an alias, remove the next one
+            # *unless* the last alias has a no-arg flag version, in which
+            # case, don't swallow the next arg if it's also a flag:
+            if not (was_flag and a.startswith('-')):
+                stripped.remove(a)
+                continue
+        if a.startswith('-'):
+            split = a.lstrip('-').split('=')
+            name = split[0]
+            # we use startswith because argparse accepts any arg to be specified
+            # by any leading section, as long as it is unique,
+            # so `--no-br` means `--no-browser` in the notebook, etc.
+            if any(alias.startswith(name) for alias in aliases):
+                stripped.remove(a)
+                if len(split) == 1:
+                    # alias passed with arg via space
+                    swallow_next = True
+                    # could have been a flag that matches an alias, e.g. `existing`
+                    # in which case, we might not swallow the next arg
+                    was_flag = name in flags
+            elif len(split) == 1 and any(flag.startswith(name) for flag in flags):
+                # strip flag, but don't swallow next, as flags don't take args
+                stripped.remove(a)
+
+    # return shortened list
+    return stripped
+
+
+def make_ipkernel_cmd(mod='ipython_kernel', executable=None, extra_arguments=[], **kw):
+    """Build Popen command list for launching an IPython kernel.
+
+    Parameters
+    ----------
+    mod : str, optional (default 'ipython_kernel')
+        A string of an IPython module whose __main__ starts an IPython kernel
+
+    executable : str, optional (default sys.executable)
+        The Python executable to use for the kernel process.
+
+    extra_arguments : list, optional
+        A list of extra arguments to pass when executing the launch code.
+
+    Returns
+    -------
+
+    A Popen command list
+    """
+    if executable is None:
+        executable = sys.executable
+    arguments = [ executable, '-m', mod, '-f', '{connection_file}' ]
+    arguments.extend(extra_arguments)
+
+    return arguments
+
+
+def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None,
+                        independent=False,
+                        cwd=None,
+                        **kw
+                        ):
+    """ Launches a localhost kernel, binding to the specified ports.
+
+    Parameters
+    ----------
+    cmd : Popen list,
+        A string of Python code that imports and executes a kernel entry point.
+
+    stdin, stdout, stderr : optional (default None)
+        Standards streams, as defined in subprocess.Popen.
+
+    independent : bool, optional (default False)
+        If set, the kernel process is guaranteed to survive if this process
+        dies. If not set, an effort is made to ensure that the kernel is killed
+        when this process dies. Note that in this case it is still good practice
+        to kill kernels manually before exiting.
+
+    cwd : path, optional
+        The working dir of the kernel process (default: cwd of this process).
+
+    Returns
+    -------
+
+    Popen instance for the kernel subprocess
+    """
+
+    # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
+    # are invalid. Unfortunately, there is in general no way to detect whether
+    # they are valid.  The following two blocks redirect them to (temporary)
+    # pipes in certain important cases.
+
+    # If this process has been backgrounded, our stdin is invalid. Since there
+    # is no compelling reason for the kernel to inherit our stdin anyway, we'll
+    # place this one safe and always redirect.
+    redirect_in = True
+    _stdin = PIPE if stdin is None else stdin
+
+    # If this process in running on pythonw, we know that stdin, stdout, and
+    # stderr are all invalid.
+    redirect_out = sys.executable.endswith('pythonw.exe')
+    if redirect_out:
+        blackhole = open(os.devnull, 'w')
+        _stdout = blackhole if stdout is None else stdout
+        _stderr = blackhole if stderr is None else stderr
+    else:
+        _stdout, _stderr = stdout, stderr
+
+    env = env if (env is not None) else os.environ.copy()
+
+    encoding = getdefaultencoding(prefer_stream=False)
+    kwargs = dict(
+        stdin=_stdin,
+        stdout=_stdout,
+        stderr=_stderr,
+        cwd=cwd,
+        env=env,
+    )
+
+    # Spawn a kernel.
+    if sys.platform == 'win32':
+        # Popen on Python 2 on Windows cannot handle unicode args or cwd
+        cmd = [ cast_bytes_py2(c, encoding) for c in cmd ]
+        if cwd:
+            cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii')
+            kwargs['cwd'] = cwd
+
+        from jupyter_client.parentpoller import ParentPollerWindows
+        # Create a Win32 event for interrupting the kernel
+        # and store it in an environment variable.
+        interrupt_event = ParentPollerWindows.create_interrupt_event()
+        env["JPY_INTERRUPT_EVENT"] = str(interrupt_event)
+        # deprecated old env name:
+        env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"]
+
+        try:
+            from _winapi import DuplicateHandle, GetCurrentProcess, \
+                DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP
+        except:
+            from _subprocess import DuplicateHandle, GetCurrentProcess, \
+                DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP
+        # Launch the kernel process
+        if independent:
+            kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP
+        else:
+            pid = GetCurrentProcess()
+            handle = DuplicateHandle(pid, pid, pid, 0,
+                                     True, # Inheritable by new processes.
+                                     DUPLICATE_SAME_ACCESS)
+            env['JPY_PARENT_PID'] = str(int(handle))
+
+        proc = Popen(cmd, **kwargs)
+
+        # Attach the interrupt event to the Popen objet so it can be used later.
+        proc.win32_interrupt_event = interrupt_event
+
+    else:
+        if independent:
+            kwargs['preexec_fn'] = lambda: os.setsid()
+        else:
+            env['JPY_PARENT_PID'] = str(os.getpid())
+
+        proc = Popen(cmd, **kwargs)
+
+    # Clean up pipes created to work around Popen bug.
+    if redirect_in:
+        if stdin is None:
+            proc.stdin.close()
+
+    return proc
+
+__all__ = [
+    'swallow_argv',
+    'make_ipkernel_cmd',
+    'launch_kernel',
+]

IPython/consoleapp.py

@@ -1,330 +1,331 b''
 """ A minimal application base mixin for all ZMQ based IPython frontends.
 
 This is not a complete console app, as subprocess will not be able to receive
 input, there is no real readline support, among other limitations. This is a
 refactoring of what used to be the IPython/qt/console/qtconsoleapp.py
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import atexit
 import os
 import signal
 import sys
 import uuid
 
 
 from IPython.config.application import boolean_flag
 from IPython.core.profiledir import ProfileDir
 from IPython.kernel.blocking import BlockingKernelClient
 from IPython.kernel import KernelManager
 from IPython.kernel import tunnel_to_kernel, find_connection_file
 from IPython.kernel.kernelspec import NoSuchKernel
 from IPython.utils.path import filefind
 from IPython.utils.traitlets import (
     Dict, List, Unicode, CUnicode, CBool, Any
 )
 from IPython.kernel.zmq.session import Session
-from IPython.kernel.connect import ConnectionFileMixin
+from IPython.kernel import connect
+ConnectionFileMixin = connect.ConnectionFileMixin
 
 from IPython.utils.localinterfaces import localhost
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = {}
 
 # the flags that are specific to the frontend
 # these must be scrubbed before being passed to the kernel,
 # or it will raise an error on unrecognized flags
 app_flags = {
     'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}},
             "Connect to an existing kernel. If no argument specified, guess most recent"),
 }
 app_flags.update(boolean_flag(
     'confirm-exit', 'IPythonConsoleApp.confirm_exit',
     """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
        to force a direct exit without any confirmation.
     """,
     """Don't prompt the user when exiting. This will terminate the kernel
        if it is owned by the frontend, and leave it alive if it is external.
     """
 ))
 flags.update(app_flags)
 
 aliases = {}
 
 # also scrub aliases from the frontend
 app_aliases = dict(
     ip = 'IPythonConsoleApp.ip',
     transport = 'IPythonConsoleApp.transport',
     hb = 'IPythonConsoleApp.hb_port',
     shell = 'IPythonConsoleApp.shell_port',
     iopub = 'IPythonConsoleApp.iopub_port',
     stdin = 'IPythonConsoleApp.stdin_port',
     existing = 'IPythonConsoleApp.existing',
     f = 'IPythonConsoleApp.connection_file',
 
     kernel = 'IPythonConsoleApp.kernel_name',
 
     ssh = 'IPythonConsoleApp.sshserver',
 )
 aliases.update(app_aliases)
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 classes = [KernelManager, ProfileDir, Session]
 
 class IPythonConsoleApp(ConnectionFileMixin):
     name = 'ipython-console-mixin'
 
     description = """
         The IPython Mixin Console.
         
         This class contains the common portions of console client (QtConsole,
         ZMQ-based terminal console, etc).  It is not a full console, in that
         launched terminal subprocesses will not be able to accept input.
         
         The Console using this mixing supports various extra features beyond
         the single-process Terminal IPython shell, such as connecting to
         existing kernel, via:
         
             ipython <appname> --existing
         
         as well as tunnel via SSH
         
     """
 
     classes = classes
     flags = Dict(flags)
     aliases = Dict(aliases)
     kernel_manager_class = KernelManager
     kernel_client_class = BlockingKernelClient
 
     kernel_argv = List(Unicode)
     # frontend flags&aliases to be stripped when building kernel_argv
     frontend_flags = Any(app_flags)
     frontend_aliases = Any(app_aliases)
 
     # create requested profiles by default, if they don't exist:
     auto_create = CBool(True)
     # connection info:
     
     sshserver = Unicode('', config=True,
         help="""The SSH server to use to connect to the kernel.""")
     sshkey = Unicode('', config=True,
         help="""Path to the ssh key to use for logging in to the ssh server.""")
     
     def _connection_file_default(self):
         return 'kernel-%i.json' % os.getpid()
 
     existing = CUnicode('', config=True,
         help="""Connect to an already running kernel""")
 
     kernel_name = Unicode('python', config=True,
         help="""The name of the default kernel to start.""")
 
     confirm_exit = CBool(True, config=True,
         help="""
         Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
         to force a direct exit without any confirmation.""",
     )
     
     def build_kernel_argv(self, argv=None):
         """build argv to be passed to kernel subprocess
         
         Override in subclasses if any args should be passed to the kernel
         """
         self.kernel_argv = self.extra_args
     
     def init_connection_file(self):
         """find the connection file, and load the info if found.
         
         The current working directory and the current profile's security
         directory will be searched for the file if it is not given by
         absolute path.
         
         When attempting to connect to an existing kernel and the `--existing`
         argument does not match an existing file, it will be interpreted as a
         fileglob, and the matching file in the current profile's security dir
         with the latest access time will be used.
         
         After this method is called, self.connection_file contains the *full path*
         to the connection file, never just its name.
         """
         if self.existing:
             try:
                 cf = find_connection_file(self.existing)
             except Exception:
                 self.log.critical("Could not find existing kernel connection file %s", self.existing)
                 self.exit(1)
             self.log.debug("Connecting to existing kernel: %s" % cf)
             self.connection_file = cf
         else:
             # not existing, check if we are going to write the file
             # and ensure that self.connection_file is a full path, not just the shortname
             try:
                 cf = find_connection_file(self.connection_file)
             except Exception:
                 # file might not exist
                 if self.connection_file == os.path.basename(self.connection_file):
                     # just shortname, put it in security dir
                     cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
                 else:
                     cf = self.connection_file
                 self.connection_file = cf
         try:
             self.connection_file = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
         except IOError:
             self.log.debug("Connection File not found: %s", self.connection_file)
             return
         
         # should load_connection_file only be used for existing?
         # as it is now, this allows reusing ports if an existing
         # file is requested
         try:
             self.load_connection_file()
         except Exception:
             self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
             self.exit(1)
     
     def init_ssh(self):
         """set up ssh tunnels, if needed."""
         if not self.existing or (not self.sshserver and not self.sshkey):
             return
         self.load_connection_file()
         
         transport = self.transport
         ip = self.ip
         
         if transport != 'tcp':
             self.log.error("Can only use ssh tunnels with TCP sockets, not %s", transport)
             sys.exit(-1)
         
         if self.sshkey and not self.sshserver:
             # specifying just the key implies that we are connecting directly
             self.sshserver = ip
             ip = localhost()
         
         # build connection dict for tunnels:
         info = dict(ip=ip,
                     shell_port=self.shell_port,
                     iopub_port=self.iopub_port,
                     stdin_port=self.stdin_port,
                     hb_port=self.hb_port
         )
         
         self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver))
         
         # tunnels return a new set of ports, which will be on localhost:
         self.ip = localhost()
         try:
             newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
         except:
             # even catch KeyboardInterrupt
             self.log.error("Could not setup tunnels", exc_info=True)
             self.exit(1)
         
         self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
         
         cf = self.connection_file
         base,ext = os.path.splitext(cf)
         base = os.path.basename(base)
         self.connection_file = os.path.basename(base)+'-ssh'+ext
         self.log.info("To connect another client via this tunnel, use:")
         self.log.info("--existing %s" % self.connection_file)
     
     def _new_connection_file(self):
         cf = ''
         while not cf:
             # we don't need a 128b id to distinguish kernels, use more readable
             # 48b node segment (12 hex chars).  Users running more than 32k simultaneous
             # kernels can subclass.
             ident = str(uuid.uuid4()).split('-')[-1]
             cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident)
             # only keep if it's actually new.  Protect against unlikely collision
             # in 48b random search space
             cf = cf if not os.path.exists(cf) else ''
         return cf
 
     def init_kernel_manager(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.
         if self.existing:
             self.kernel_manager = None
             return
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
         # Create a KernelManager and start a kernel.
         try:
             self.kernel_manager = self.kernel_manager_class(
                                     ip=self.ip,
                                     session=self.session,
                                     transport=self.transport,
                                     shell_port=self.shell_port,
                                     iopub_port=self.iopub_port,
                                     stdin_port=self.stdin_port,
                                     hb_port=self.hb_port,
                                     connection_file=self.connection_file,
                                     kernel_name=self.kernel_name,
                                     parent=self,
                                     ipython_dir=self.ipython_dir,
             )
         except NoSuchKernel:
             self.log.critical("Could not find kernel %s", self.kernel_name)
             self.exit(1)
 
         self.kernel_manager.client_factory = self.kernel_client_class
         # FIXME: remove special treatment of IPython kernels
         kwargs = {}
         if self.kernel_manager.ipython_kernel:
             kwargs['extra_arguments'] = self.kernel_argv
         self.kernel_manager.start_kernel(**kwargs)
         atexit.register(self.kernel_manager.cleanup_ipc_files)
 
         if self.sshserver:
             # ssh, write new connection file
             self.kernel_manager.write_connection_file()
 
         # in case KM defaults / ssh writing changes things:
         km = self.kernel_manager
         self.shell_port=km.shell_port
         self.iopub_port=km.iopub_port
         self.stdin_port=km.stdin_port
         self.hb_port=km.hb_port
         self.connection_file = km.connection_file
 
         atexit.register(self.kernel_manager.cleanup_connection_file)
 
     def init_kernel_client(self):
         if self.kernel_manager is not None:
             self.kernel_client = self.kernel_manager.client()
         else:
             self.kernel_client = self.kernel_client_class(
                                 session=self.session,
                                 ip=self.ip,
                                 transport=self.transport,
                                 shell_port=self.shell_port,
                                 iopub_port=self.iopub_port,
                                 stdin_port=self.stdin_port,
                                 hb_port=self.hb_port,
                                 connection_file=self.connection_file,
                                 parent=self,
             )
 
         self.kernel_client.start_channels()
 
 
 
     def initialize(self, argv=None):
         """
         Classes which mix this class in should call:
                IPythonConsoleApp.initialize(self,argv)
         """
         self.init_connection_file()
         self.init_ssh()
         self.init_kernel_manager()
         self.init_kernel_client()
 

IPython/kernel/__init__.py

@@ -1,67 +1,29 b''
-"""IPython kernels and associated utilities
-
-For connecting to kernels, use jupyter_client
 """
-
+Shim to maintain backwards compatibility with old IPython.kernel imports.
+"""
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
-# Shim to maintain backwards compatibility with old IPython.kernel imports.
-
 import sys
 from warnings import warn
 
 warn("The `IPython.kernel` package has been deprecated. "
      "You should import from ipython_kernel or jupyter_client instead.")
 
-from IPython.utils.shimmodule import ShimModule
-
-# Shims for jupyter_client
-# Can't do a single shim, because the package didn't move all together
 
-for name in (
-    'adapter',
-    'blocking',
-    'channels',
-    'channelsabc',
-    'client',
-    'clientabc',
-    'connect',
-    'ioloop',
-    'kernelspec',
-    'kernelspecapp',
-    'launcher',
-    'manager',
-    'managerabc',
-    'multikernelmanager',
-    'restarter',
-    'threaded',
-    'tests.test_adapter',
-    'tests.test_connect',
-    'tests.test_kernelmanager',
-    'tests.test_kernelspec',
-    'tests.test_launcher',
-    'tests.test_multikernelmanager',
-    'tests.test_public_api',
-):
-    sys.modules['IPython.kernel.%s' % name] = \
-        ShimModule(name, mirror='jupyter_client.%s' % name)
+from IPython.utils.shimmodule import ShimModule
 
-# some files moved out of the zmq prefix
-for name in (
-    'session',
-    'tests.test_session',
-):
-    sys.modules['IPython.kernel.zmq.%s' % name] = \
-        ShimModule(name, mirror='jupyter_client.%s' % name)
-# preserve top-level API modules, all from jupyter_client
+# session moved relative to top-level
+sys.modules['IPython.kernel.zmq.session'] = ShimModule('session', mirror='jupyter_client.session')
 
-# just for friendlier zmq version check
-from . import zmq
+for pkg in ('comm', 'inprocess', 'resources', 'zmq'):
+    sys.modules['IPython.kernel.%s' % pkg] = ShimModule(pkg, mirror='ipython_kernel.%s' % pkg)
+for pkg in ('ioloop', 'blocking'):
+    sys.modules['IPython.kernel.%s' % pkg] = ShimModule(pkg, mirror='jupyter_client.%s' % pkg)
 
-from jupyter_client.connect import *
-from jupyter_client.launcher import *
-from jupyter_client.client import KernelClient
-from jupyter_client.manager import KernelManager, run_kernel
-from jupyter_client.blocking import BlockingKernelClient
-from jupyter_client.multikernelmanager import MultiKernelManager
+# required for `from IPython.kernel import PKG`
+from ipython_kernel import comm, inprocess, resources, zmq
+from jupyter_client import ioloop, blocking
+# public API
+from ipython_kernel.connect import *
+from jupyter_client import *

IPython/testing/iptest.py

@@ -1,518 +1,520 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import glob
 from io import BytesIO
 import os
 import os.path as path
 import sys
 from threading import Thread, Lock, Event
 import warnings
 
 import nose.plugins.builtin
 from nose.plugins.xunit import Xunit
 from nose import SkipTest
 from nose.core import TestProgram
 from nose.plugins import Plugin
 from nose.util import safe_str
 
 from IPython.utils.process import is_cmd_found
 from IPython.utils.py3compat import bytes_to_str
 from IPython.utils.importstring import import_item
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 from IPython.external.decorators import KnownFailure, knownfailureif
 
 pjoin = path.join
 
 #-----------------------------------------------------------------------------
 # Warnings control
 #-----------------------------------------------------------------------------
 
 # Twisted generates annoying warnings with Python 2.6, as will do other code
 # that imports 'sets' as of today
 warnings.filterwarnings('ignore', 'the sets module is deprecated',
                         DeprecationWarning )
 
 # This one also comes from Twisted
 warnings.filterwarnings('ignore', 'the sha module is deprecated',
                         DeprecationWarning)
 
 # Wx on Fedora11 spits these out
 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
                         UserWarning)
 
 # ------------------------------------------------------------------------------
 # Monkeypatch Xunit to count known failures as skipped.
 # ------------------------------------------------------------------------------
 def monkeypatch_xunit():
     try:
         knownfailureif(True)(lambda: None)()
     except Exception as e:
         KnownFailureTest = type(e)
 
     def addError(self, test, err, capt=None):
         if issubclass(err[0], KnownFailureTest):
             err = (SkipTest,) + err[1:]
         return self.orig_addError(test, err, capt)
 
     Xunit.orig_addError = Xunit.addError
     Xunit.addError = addError
 
 #-----------------------------------------------------------------------------
 # Check which dependencies are installed and greater than minimum version.
 #-----------------------------------------------------------------------------
 def extract_version(mod):
     return mod.__version__
 
 def test_for(item, min_version=None, callback=extract_version):
     """Test to see if item is importable, and optionally check against a minimum
     version.
 
     If min_version is given, the default behavior is to check against the
     `__version__` attribute of the item, but specifying `callback` allows you to
     extract the value you are interested in. e.g::
 
         In [1]: import sys
 
         In [2]: from IPython.testing.iptest import test_for
 
         In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
         Out[3]: True
 
     """
     try:
         check = import_item(item)
     except (ImportError, RuntimeError):
         # GTK reports Runtime error if it can't be initialized even if it's
         # importable.
         return False
     else:
         if min_version:
             if callback:
                 # extra processing step to get version to compare
                 check = callback(check)
 
             return check >= min_version
         else:
             return True
 
 # Global dict where we can store information on what we have and what we don't
 # have available at test run time
 have = {}
 
 have['curses'] = test_for('_curses')
 have['matplotlib'] = test_for('matplotlib')
 have['numpy'] = test_for('numpy')
 have['pexpect'] = test_for('pexpect')
 have['pymongo'] = test_for('pymongo')
 have['pygments'] = test_for('pygments')
 have['qt'] = test_for('IPython.external.qt')
 have['sqlite3'] = test_for('sqlite3')
 have['tornado'] = test_for('tornado.version_info', (4,0), callback=None)
 have['jinja2'] = test_for('jinja2')
 have['mistune'] = test_for('mistune')
 have['requests'] = test_for('requests')
 have['sphinx'] = test_for('sphinx')
 have['jsonschema'] = test_for('jsonschema')
 have['terminado'] = test_for('terminado')
 have['casperjs'] = is_cmd_found('casperjs')
 have['phantomjs'] = is_cmd_found('phantomjs')
 have['slimerjs'] = is_cmd_found('slimerjs')
 
 min_zmq = (13,)
 
 have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x())
 
 #-----------------------------------------------------------------------------
 # Test suite definitions
 #-----------------------------------------------------------------------------
 
 test_group_names = ['parallel', 'kernel', 'kernel.inprocess', 'config', 'core',
                     'extensions', 'lib', 'terminal', 'testing', 'utils',
                     'nbformat', 'qt', 'html', 'nbconvert'
                    ]
 
 class TestSection(object):
     def __init__(self, name, includes):
         self.name = name
         self.includes = includes
         self.excludes = []
         self.dependencies = []
         self.enabled = True
     
     def exclude(self, module):
         if not module.startswith('IPython'):
             module = self.includes[0] + "." + module
         self.excludes.append(module.replace('.', os.sep))
     
     def requires(self, *packages):
         self.dependencies.extend(packages)
     
     @property
     def will_run(self):
         return self.enabled and all(have[p] for p in self.dependencies)
 
 shims = {
     'parallel': 'ipython_parallel',
+    'kernel': 'ipython_kernel',
+    'kernel.inprocess': 'ipython_kernel.inprocess',
 }
 
 # Name -> (include, exclude, dependencies_met)
 test_sections = {n:TestSection(n, [shims.get(n, 'IPython.%s' % n)]) for n in test_group_names}
 
 
 # Exclusions and dependencies
 # ---------------------------
 
 # core:
 sec = test_sections['core']
 if not have['sqlite3']:
     sec.exclude('tests.test_history')
     sec.exclude('history')
 if not have['matplotlib']:
     sec.exclude('pylabtools'),
     sec.exclude('tests.test_pylabtools')
 
 # lib:
 sec = test_sections['lib']
 if not have['zmq']:
     sec.exclude('kernel')
 # We do this unconditionally, so that the test suite doesn't import
 # gtk, changing the default encoding and masking some unicode bugs.
 sec.exclude('inputhookgtk')
 # We also do this unconditionally, because wx can interfere with Unix signals.
 # There are currently no tests for it anyway.
 sec.exclude('inputhookwx')
 # Testing inputhook will need a lot of thought, to figure out
 # how to have tests that don't lock up with the gui event
 # loops in the picture
 sec.exclude('inputhook')
 
 # testing:
 sec = test_sections['testing']
 # These have to be skipped on win32 because they use echo, rm, cd, etc.
 # See ticket https://github.com/ipython/ipython/issues/87
 if sys.platform == 'win32':
     sec.exclude('plugin.test_exampleip')
     sec.exclude('plugin.dtexample')
 
 # terminal:
 if (not have['pexpect']) or (not have['zmq']):
     test_sections['terminal'].exclude('console')
 
 # parallel
 sec = test_sections['parallel']
 sec.requires('zmq')
 if not have['pymongo']:
     sec.exclude('controller.mongodb')
     sec.exclude('tests.test_mongodb')
 
 # kernel:
 sec = test_sections['kernel']
 sec.requires('zmq')
 # The in-process kernel tests are done in a separate section
 sec.exclude('inprocess')
 # importing gtk sets the default encoding, which we want to avoid
 sec.exclude('zmq.gui.gtkembed')
 sec.exclude('zmq.gui.gtk3embed')
 if not have['matplotlib']:
     sec.exclude('zmq.pylab')
 
 # kernel.inprocess:
 test_sections['kernel.inprocess'].requires('zmq')
 
 # extensions:
 sec = test_sections['extensions']
 # This is deprecated in favour of rpy2
 sec.exclude('rmagic')
 # autoreload does some strange stuff, so move it to its own test section
 sec.exclude('autoreload')
 sec.exclude('tests.test_autoreload')
 test_sections['autoreload'] = TestSection('autoreload',
         ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'])
 test_group_names.append('autoreload')
 
 # qt:
 test_sections['qt'].requires('zmq', 'qt', 'pygments')
 
 # html:
 sec = test_sections['html']
 sec.requires('zmq', 'tornado', 'requests', 'sqlite3', 'jsonschema')
 # The notebook 'static' directory contains JS, css and other
 # files for web serving.  Occasionally projects may put a .py
 # file in there (MathJax ships a conf.py), so we might as
 # well play it safe and skip the whole thing.
 sec.exclude('static')
 sec.exclude('tasks')
 if not have['jinja2']:
     sec.exclude('notebookapp')
 if not have['pygments'] or not have['jinja2']:
     sec.exclude('nbconvert')
 if not have['terminado']:
     sec.exclude('terminal')
 
 # config:
 # Config files aren't really importable stand-alone
 test_sections['config'].exclude('profile')
 
 # nbconvert:
 sec = test_sections['nbconvert']
 sec.requires('pygments', 'jinja2', 'jsonschema', 'mistune')
 # Exclude nbconvert directories containing config files used to test.
 # Executing the config files with iptest would cause an exception.
 sec.exclude('tests.files')
 sec.exclude('exporters.tests.files')
 if not have['tornado']:
     sec.exclude('nbconvert.post_processors.serve')
     sec.exclude('nbconvert.post_processors.tests.test_serve')
 
 # nbformat:
 test_sections['nbformat'].requires('jsonschema')
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def check_exclusions_exist():
     from IPython.utils.path import get_ipython_package_dir
     from IPython.utils.warn import warn
     parent = os.path.dirname(get_ipython_package_dir())
     for sec in test_sections:
         for pattern in sec.exclusions:
             fullpath = pjoin(parent, pattern)
             if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
                 warn("Excluding nonexistent file: %r" % pattern)
 
 
 class ExclusionPlugin(Plugin):
     """A nose plugin to effect our exclusions of files and directories.
     """
     name = 'exclusions'
     score = 3000  # Should come before any other plugins
     
     def __init__(self, exclude_patterns=None):
         """
         Parameters
         ----------
 
         exclude_patterns : sequence of strings, optional
           Filenames containing these patterns (as raw strings, not as regular
           expressions) are excluded from the tests.
         """
         self.exclude_patterns = exclude_patterns or []
         super(ExclusionPlugin, self).__init__()
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
     
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Override nose trying to disable plugin.
         self.enabled = True
         
     def wantFile(self, filename):
         """Return whether the given filename should be scanned for tests.
         """
         if any(pat in filename for pat in self.exclude_patterns):
             return False
         return None
 
     def wantDirectory(self, directory):
         """Return whether the given directory should be scanned for tests.
         """
         if any(pat in directory for pat in self.exclude_patterns):
             return False
         return None
 
 
 class StreamCapturer(Thread):
     daemon = True  # Don't hang if main thread crashes
     started = False
     def __init__(self, echo=False):
         super(StreamCapturer, self).__init__()
         self.echo = echo
         self.streams = []
         self.buffer = BytesIO()
         self.readfd, self.writefd = os.pipe()
         self.buffer_lock = Lock()
         self.stop = Event()
 
     def run(self):
         self.started = True
 
         while not self.stop.is_set():
             chunk = os.read(self.readfd, 1024)
 
             with self.buffer_lock:
                 self.buffer.write(chunk)
             if self.echo:
                 sys.stdout.write(bytes_to_str(chunk))
     
         os.close(self.readfd)
         os.close(self.writefd)
 
     def reset_buffer(self):
         with self.buffer_lock:
             self.buffer.truncate(0)
             self.buffer.seek(0)
 
     def get_buffer(self):
         with self.buffer_lock:
             return self.buffer.getvalue()
 
     def ensure_started(self):
         if not self.started:
             self.start()
 
     def halt(self):
         """Safely stop the thread."""
         if not self.started:
             return
 
         self.stop.set()
         os.write(self.writefd, b'\0')  # Ensure we're not locked in a read()
         self.join()
 
 class SubprocessStreamCapturePlugin(Plugin):
     name='subprocstreams'
     def __init__(self):
         Plugin.__init__(self)
         self.stream_capturer = StreamCapturer()
         self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture')
         # This is ugly, but distant parts of the test machinery need to be able
         # to redirect streams, so we make the object globally accessible.
         nose.iptest_stdstreams_fileno = self.get_write_fileno
 
     def get_write_fileno(self):
         if self.destination == 'capture':
             self.stream_capturer.ensure_started()
             return self.stream_capturer.writefd
         elif self.destination == 'discard':
             return os.open(os.devnull, os.O_WRONLY)
         else:
             return sys.__stdout__.fileno()
     
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Override nose trying to disable plugin.
         if self.destination == 'capture':
             self.enabled = True
     
     def startTest(self, test):
         # Reset log capture
         self.stream_capturer.reset_buffer()
     
     def formatFailure(self, test, err):
         # Show output
         ec, ev, tb = err
         captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
         if captured.strip():
             ev = safe_str(ev)
             out = [ev, '>> begin captured subprocess output <<',
                     captured,
                     '>> end captured subprocess output <<']
             return ec, '\n'.join(out), tb
 
         return err
     
     formatError = formatFailure
     
     def finalize(self, result):
         self.stream_capturer.halt()
 
 
 def run_iptest():
     """Run the IPython test suite using nose.
 
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
     # Apply our monkeypatch to Xunit
     if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
         monkeypatch_xunit()
 
     warnings.filterwarnings('ignore',
         'This will be removed soon.  Use IPython.testing.util instead')
     
     arg1 = sys.argv[1]
     if arg1 in test_sections:
         section = test_sections[arg1]
         sys.argv[1:2] = section.includes
     elif arg1.startswith('IPython.') and arg1[8:] in test_sections:
         section = test_sections[arg1[8:]]
         sys.argv[1:2] = section.includes
     else:
         section = TestSection(arg1, includes=[arg1])
         
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
 
                         '--with-ipdoctest',
                         '--ipdoctest-tests','--ipdoctest-extension=txt',
 
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
     if '-a' not in argv and '-A' not in argv:
         argv = argv + ['-a', '!crash']
 
     if nose.__version__ >= '0.11':
         # I don't fully understand why we need this one, but depending on what
         # directory the test suite is run from, if we don't give it, 0 tests
         # get run.  Specifically, if the test suite is run from the source dir
         # with an argument (like 'iptest.py IPython.core', 0 tests are run,
         # even if the same call done in this directory works fine).  It appears
         # that if the requested package is in the current dir, nose bails early
         # by default.  Since it's otherwise harmless, leave it in by default
         # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
         argv.append('--traverse-namespace')
 
     # use our plugin for doctesting.  It will remove the standard doctest plugin
     # if it finds it enabled
     plugins = [ExclusionPlugin(section.excludes), IPythonDoctest(), KnownFailure(),
                SubprocessStreamCapturePlugin() ]
     
     # Use working directory set by parent process (see iptestcontroller)
     if 'IPTEST_WORKING_DIR' in os.environ:
         os.chdir(os.environ['IPTEST_WORKING_DIR'])
     
     # We need a global ipython running in this process, but the special
     # in-process group spawns its own IPython kernels, so for *that* group we
     # must avoid also opening the global one (otherwise there's a conflict of
     # singletons).  Ultimately the solution to this problem is to refactor our
     # assumptions about what needs to be a singleton and what doesn't (app
     # objects should, individual shells shouldn't).  But for now, this
     # workaround allows the test suite for the inprocess module to complete.
     if 'kernel.inprocess' not in section.name:
         from IPython.testing import globalipapp
         globalipapp.start_ipython()
 
     # Now nose can run
     TestProgram(argv=argv, addplugins=plugins)
 
 if __name__ == '__main__':
     run_iptest()
 

jupyter_client/launcher.py

@@ -1,226 +1,226 b''
 """Utilities for launching kernels
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 from subprocess import Popen, PIPE
 
 from IPython.utils.encoding import getdefaultencoding
 from IPython.utils.py3compat import cast_bytes_py2
 
 
 def swallow_argv(argv, aliases=None, flags=None):
     """strip frontend-specific aliases and flags from an argument list
 
     For use primarily in frontend apps that want to pass a subset of command-line
     arguments through to a subprocess, where frontend-specific flags and aliases
     should be removed from the list.
 
     Parameters
     ----------
 
     argv : list(str)
         The starting argv, to be filtered
     aliases : container of aliases (dict, list, set, etc.)
         The frontend-specific aliases to be removed
     flags : container of flags (dict, list, set, etc.)
         The frontend-specific flags to be removed
 
     Returns
     -------
 
     argv : list(str)
         The argv list, excluding flags and aliases that have been stripped
     """
 
     if aliases is None:
         aliases = set()
     if flags is None:
         flags = set()
 
     stripped = list(argv) # copy
 
     swallow_next = False
     was_flag = False
     for a in argv:
         if a == '--':
             break
         if swallow_next:
             swallow_next = False
             # last arg was an alias, remove the next one
             # *unless* the last alias has a no-arg flag version, in which
             # case, don't swallow the next arg if it's also a flag:
             if not (was_flag and a.startswith('-')):
                 stripped.remove(a)
                 continue
         if a.startswith('-'):
             split = a.lstrip('-').split('=')
             name = split[0]
             # we use startswith because argparse accepts any arg to be specified
             # by any leading section, as long as it is unique,
             # so `--no-br` means `--no-browser` in the notebook, etc.
             if any(alias.startswith(name) for alias in aliases):
                 stripped.remove(a)
                 if len(split) == 1:
                     # alias passed with arg via space
                     swallow_next = True
                     # could have been a flag that matches an alias, e.g. `existing`
                     # in which case, we might not swallow the next arg
                     was_flag = name in flags
             elif len(split) == 1 and any(flag.startswith(name) for flag in flags):
                 # strip flag, but don't swallow next, as flags don't take args
                 stripped.remove(a)
 
     # return shortened list
     return stripped
 
 
-def make_ipkernel_cmd(mod='IPython.kernel', executable=None, extra_arguments=[], **kw):
+def make_ipkernel_cmd(mod='ipython_kernel', executable=None, extra_arguments=[], **kw):
     """Build Popen command list for launching an IPython kernel.
 
     Parameters
     ----------
-    mod : str, optional (default 'IPython.kernel')
+    mod : str, optional (default 'ipython_kernel')
         A string of an IPython module whose __main__ starts an IPython kernel
 
     executable : str, optional (default sys.executable)
         The Python executable to use for the kernel process.
 
     extra_arguments : list, optional
         A list of extra arguments to pass when executing the launch code.
 
     Returns
     -------
 
     A Popen command list
     """
     if executable is None:
         executable = sys.executable
     arguments = [ executable, '-m', mod, '-f', '{connection_file}' ]
     arguments.extend(extra_arguments)
 
     return arguments
 
 
 def launch_kernel(cmd, stdin=None, stdout=None, stderr=None, env=None,
                         independent=False,
                         cwd=None,
                         **kw
                         ):
     """ Launches a localhost kernel, binding to the specified ports.
 
     Parameters
     ----------
     cmd : Popen list,
         A string of Python code that imports and executes a kernel entry point.
 
     stdin, stdout, stderr : optional (default None)
         Standards streams, as defined in subprocess.Popen.
 
     independent : bool, optional (default False)
         If set, the kernel process is guaranteed to survive if this process
         dies. If not set, an effort is made to ensure that the kernel is killed
         when this process dies. Note that in this case it is still good practice
         to kill kernels manually before exiting.
 
     cwd : path, optional
         The working dir of the kernel process (default: cwd of this process).
 
     Returns
     -------
 
     Popen instance for the kernel subprocess
     """
 
     # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
     # are invalid. Unfortunately, there is in general no way to detect whether
     # they are valid.  The following two blocks redirect them to (temporary)
     # pipes in certain important cases.
 
     # If this process has been backgrounded, our stdin is invalid. Since there
     # is no compelling reason for the kernel to inherit our stdin anyway, we'll
     # place this one safe and always redirect.
     redirect_in = True
     _stdin = PIPE if stdin is None else stdin
 
     # If this process in running on pythonw, we know that stdin, stdout, and
     # stderr are all invalid.
     redirect_out = sys.executable.endswith('pythonw.exe')
     if redirect_out:
         blackhole = open(os.devnull, 'w')
         _stdout = blackhole if stdout is None else stdout
         _stderr = blackhole if stderr is None else stderr
     else:
         _stdout, _stderr = stdout, stderr
 
     env = env if (env is not None) else os.environ.copy()
 
     encoding = getdefaultencoding(prefer_stream=False)
     kwargs = dict(
         stdin=_stdin,
         stdout=_stdout,
         stderr=_stderr,
         cwd=cwd,
         env=env,
     )
 
     # Spawn a kernel.
     if sys.platform == 'win32':
         # Popen on Python 2 on Windows cannot handle unicode args or cwd
         cmd = [ cast_bytes_py2(c, encoding) for c in cmd ]
         if cwd:
             cwd = cast_bytes_py2(cwd, sys.getfilesystemencoding() or 'ascii')
             kwargs['cwd'] = cwd
 
         from jupyter_client.parentpoller import ParentPollerWindows
         # Create a Win32 event for interrupting the kernel
         # and store it in an environment variable.
         interrupt_event = ParentPollerWindows.create_interrupt_event()
         env["JPY_INTERRUPT_EVENT"] = str(interrupt_event)
         # deprecated old env name:
         env["IPY_INTERRUPT_EVENT"] = env["JPY_INTERRUPT_EVENT"]
 
         try:
             from _winapi import DuplicateHandle, GetCurrentProcess, \
                 DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP
         except:
             from _subprocess import DuplicateHandle, GetCurrentProcess, \
                 DUPLICATE_SAME_ACCESS, CREATE_NEW_PROCESS_GROUP
         # Launch the kernel process
         if independent:
             kwargs['creationflags'] = CREATE_NEW_PROCESS_GROUP
         else:
             pid = GetCurrentProcess()
             handle = DuplicateHandle(pid, pid, pid, 0,
                                      True, # Inheritable by new processes.
                                      DUPLICATE_SAME_ACCESS)
             env['JPY_PARENT_PID'] = str(int(handle))
 
         proc = Popen(cmd, **kwargs)
 
         # Attach the interrupt event to the Popen objet so it can be used later.
         proc.win32_interrupt_event = interrupt_event
 
     else:
         if independent:
             kwargs['preexec_fn'] = lambda: os.setsid()
         else:
             env['JPY_PARENT_PID'] = str(os.getpid())
 
         proc = Popen(cmd, **kwargs)
 
     # Clean up pipes created to work around Popen bug.
     if redirect_in:
         if stdin is None:
             proc.stdin.close()
 
     return proc
 
 __all__ = [
     'swallow_argv',
     'make_ipkernel_cmd',
     'launch_kernel',
 ]