Merge pull request #864 from ipython/termzmq

Two-process terminal frontend: this branch adds a new IPython frontend, invoked via

ipython console

that behaves much like the regular, old ipython, but runs over zeromq in two processes.  This means that such a client can connect to existing kernels initiated by the Qt console, the notebook or standalone (i.e. via `ipython kernel`).

We still have some internal architectural cleanups to perform to simplify how the various frontends talk to the kernels, but by having this main piece in, the complete picture is clearer, and that refactoring work can be carried post-0.12.

This frontend should still be considered experimental.

IPython/frontend/consoleapp.py

@@ -0,0 +1,352
+""" A minimal application base mixin for all ZMQ based IPython frontends.
+
+This is not a complete console app, as subprocess will not be able to receive
+input, there is no real readline support, among other limitations. This is a
+refactoring of what used to be the IPython/frontend/qt/console/qtconsoleapp.py
+
+Authors:
+
+* Evan Patterson
+* Min RK
+* Erik Tollerud
+* Fernando Perez
+* Bussonnier Matthias
+* Thomas Kluyver
+* Paul Ivanov
+
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib imports
+import atexit
+import json
+import os
+import signal
+import sys
+import uuid
+
+
+# Local imports
+from IPython.config.application import boolean_flag
+from IPython.config.configurable import Configurable
+from IPython.core.profiledir import ProfileDir
+from IPython.lib.kernel import tunnel_to_kernel, find_connection_file, swallow_argv
+from IPython.zmq.blockingkernelmanager import BlockingKernelManager
+from IPython.utils.path import filefind
+from IPython.utils.py3compat import str_to_bytes
+from IPython.utils.traitlets import (
+    Dict, List, Unicode, CUnicode, Int, CBool, Any
+)
+from IPython.zmq.ipkernel import (
+    flags as ipkernel_flags,
+    aliases as ipkernel_aliases,
+    IPKernelApp
+)
+from IPython.zmq.session import Session, default_secure
+from IPython.zmq.zmqshell import ZMQInteractiveShell
+
+#-----------------------------------------------------------------------------
+# Network Constants
+#-----------------------------------------------------------------------------
+
+from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Aliases and Flags
+#-----------------------------------------------------------------------------
+
+flags = dict(ipkernel_flags)
+
+# the flags that are specific to the frontend
+# these must be scrubbed before being passed to the kernel,
+# or it will raise an error on unrecognized flags
+app_flags = {
+    'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}},
+            "Connect to an existing kernel. If no argument specified, guess most recent"),
+}
+app_flags.update(boolean_flag(
+    'confirm-exit', 'IPythonConsoleApp.confirm_exit',
+    """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
+       to force a direct exit without any confirmation.
+    """,
+    """Don't prompt the user when exiting. This will terminate the kernel
+       if it is owned by the frontend, and leave it alive if it is external.
+    """
+))
+flags.update(app_flags)
+
+aliases = dict(ipkernel_aliases)
+
+# also scrub aliases from the frontend
+app_aliases = dict(
+    hb = 'IPythonConsoleApp.hb_port',
+    shell = 'IPythonConsoleApp.shell_port',
+    iopub = 'IPythonConsoleApp.iopub_port',
+    stdin = 'IPythonConsoleApp.stdin_port',
+    ip = 'IPythonConsoleApp.ip',
+    existing = 'IPythonConsoleApp.existing',
+    f = 'IPythonConsoleApp.connection_file',
+
+
+    ssh = 'IPythonConsoleApp.sshserver',
+)
+aliases.update(app_aliases)
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# IPythonConsole
+#-----------------------------------------------------------------------------
+
+
+class IPythonConsoleApp(Configurable):
+    name = 'ipython-console-mixin'
+    default_config_file_name='ipython_config.py'
+
+    description = """
+        The IPython Mixin Console.
+        
+        This class contains the common portions of console client (QtConsole,
+        ZMQ-based terminal console, etc).  It is not a full console, in that
+        launched terminal subprocesses will not be able to accept input.
+        
+        The Console using this mixing supports various extra features beyond
+        the single-process Terminal IPython shell, such as connecting to
+        existing kernel, via:
+        
+            ipython <appname> --existing
+        
+        as well as tunnel via SSH
+        
+    """
+
+    classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session]
+    flags = Dict(flags)
+    aliases = Dict(aliases)
+    kernel_manager_class = BlockingKernelManager
+
+    kernel_argv = List(Unicode)
+    # frontend flags&aliases to be stripped when building kernel_argv
+    frontend_flags = Any(app_flags)
+    frontend_aliases = Any(app_aliases)
+
+    pure = CBool(False, config=True,
+        help="Use a pure Python kernel instead of an IPython kernel.")
+    # create requested profiles by default, if they don't exist:
+    auto_create = CBool(True)
+    # connection info:
+    ip = Unicode(LOCALHOST, config=True,
+        help="""Set the kernel\'s IP address [default localhost].
+        If the IP address is something other than localhost, then
+        Consoles on other machines will be able to connect
+        to the Kernel, so be careful!"""
+    )
+    
+    sshserver = Unicode('', config=True,
+        help="""The SSH server to use to connect to the kernel.""")
+    sshkey = Unicode('', config=True,
+        help="""Path to the ssh key to use for logging in to the ssh server.""")
+    
+    hb_port = Int(0, config=True,
+        help="set the heartbeat port [default: random]")
+    shell_port = Int(0, config=True,
+        help="set the shell (XREP) port [default: random]")
+    iopub_port = Int(0, config=True,
+        help="set the iopub (PUB) port [default: random]")
+    stdin_port = Int(0, config=True,
+        help="set the stdin (XREQ) port [default: random]")
+    connection_file = Unicode('', config=True,
+        help="""JSON file in which to store connection info [default: kernel-<pid>.json]
+
+        This file will contain the IP, ports, and authentication key needed to connect
+        clients to this kernel. By default, this file will be created in the security-dir
+        of the current profile, but can be specified by absolute path.
+        """)
+    def _connection_file_default(self):
+        return 'kernel-%i.json' % os.getpid()
+
+    existing = CUnicode('', config=True,
+        help="""Connect to an already running kernel""")
+
+    confirm_exit = CBool(True, config=True,
+        help="""
+        Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
+        to force a direct exit without any confirmation.""",
+    )
+
+
+    def build_kernel_argv(self, argv=None):
+        """build argv to be passed to kernel subprocess"""
+        if argv is None:
+            argv = sys.argv[1:]
+        self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags)
+        # kernel should inherit default config file from frontend
+        self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
+    
+    def init_connection_file(self):
+        """find the connection file, and load the info if found.
+        
+        The current working directory and the current profile's security
+        directory will be searched for the file if it is not given by
+        absolute path.
+        
+        When attempting to connect to an existing kernel and the `--existing`
+        argument does not match an existing file, it will be interpreted as a
+        fileglob, and the matching file in the current profile's security dir
+        with the latest access time will be used.
+        
+        After this method is called, self.connection_file contains the *full path*
+        to the connection file, never just its name.
+        """
+        if self.existing:
+            try:
+                cf = find_connection_file(self.existing)
+            except Exception:
+                self.log.critical("Could not find existing kernel connection file %s", self.existing)
+                self.exit(1)
+            self.log.info("Connecting to existing kernel: %s" % cf)
+            self.connection_file = cf
+        else:
+            # not existing, check if we are going to write the file
+            # and ensure that self.connection_file is a full path, not just the shortname
+            try:
+                cf = find_connection_file(self.connection_file)
+            except Exception:
+                # file might not exist
+                if self.connection_file == os.path.basename(self.connection_file):
+                    # just shortname, put it in security dir
+                    cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
+                else:
+                    cf = self.connection_file
+                self.connection_file = cf
+        
+        # should load_connection_file only be used for existing?
+        # as it is now, this allows reusing ports if an existing
+        # file is requested
+        try:
+            self.load_connection_file()
+        except Exception:
+            self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
+            self.exit(1)
+    
+    def load_connection_file(self):
+        """load ip/port/hmac config from JSON connection file"""
+        # this is identical to KernelApp.load_connection_file
+        # perhaps it can be centralized somewhere?
+        try:
+            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+        except IOError:
+            self.log.debug("Connection File not found: %s", self.connection_file)
+            return
+        self.log.debug(u"Loading connection file %s", fname)
+        with open(fname) as f:
+            s = f.read()
+        cfg = json.loads(s)
+        if self.ip == LOCALHOST and 'ip' in cfg:
+            # not overridden by config or cl_args
+            self.ip = cfg['ip']
+        for channel in ('hb', 'shell', 'iopub', 'stdin'):
+            name = channel + '_port'
+            if getattr(self, name) == 0 and name in cfg:
+                # not overridden by config or cl_args
+                setattr(self, name, cfg[name])
+        if 'key' in cfg:
+            self.config.Session.key = str_to_bytes(cfg['key'])
+    
+    def init_ssh(self):
+        """set up ssh tunnels, if needed."""
+        if not self.sshserver and not self.sshkey:
+            return
+        
+        if self.sshkey and not self.sshserver:
+            # specifying just the key implies that we are connecting directly
+            self.sshserver = self.ip
+            self.ip = LOCALHOST
+        
+        # build connection dict for tunnels:
+        info = dict(ip=self.ip,
+                    shell_port=self.shell_port,
+                    iopub_port=self.iopub_port,
+                    stdin_port=self.stdin_port,
+                    hb_port=self.hb_port
+        )
+        
+        self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
+        
+        # tunnels return a new set of ports, which will be on localhost:
+        self.ip = LOCALHOST
+        try:
+            newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
+        except:
+            # even catch KeyboardInterrupt
+            self.log.error("Could not setup tunnels", exc_info=True)
+            self.exit(1)
+        
+        self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
+        
+        cf = self.connection_file
+        base,ext = os.path.splitext(cf)
+        base = os.path.basename(base)
+        self.connection_file = os.path.basename(base)+'-ssh'+ext
+        self.log.critical("To connect another client via this tunnel, use:")
+        self.log.critical("--existing %s" % self.connection_file)
+    
+    def _new_connection_file(self):
+        cf = ''
+        while not cf:
+            # we don't need a 128b id to distinguish kernels, use more readable
+            # 48b node segment (12 hex chars).  Users running more than 32k simultaneous
+            # kernels can subclass.
+            ident = str(uuid.uuid4()).split('-')[-1]
+            cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident)
+            # only keep if it's actually new.  Protect against unlikely collision
+            # in 48b random search space
+            cf = cf if not os.path.exists(cf) else ''
+        return cf
+
+    def init_kernel_manager(self):
+        # Don't let Qt or ZMQ swallow KeyboardInterupts.
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+        # Create a KernelManager and start a kernel.
+        self.kernel_manager = self.kernel_manager_class(
+                                ip=self.ip,
+                                shell_port=self.shell_port,
+                                iopub_port=self.iopub_port,
+                                stdin_port=self.stdin_port,
+                                hb_port=self.hb_port,
+                                connection_file=self.connection_file,
+                                config=self.config,
+        )
+        # start the kernel
+        if not self.existing:
+            kwargs = dict(ipython=not self.pure)
+            kwargs['extra_arguments'] = self.kernel_argv
+            self.kernel_manager.start_kernel(**kwargs)
+        elif self.sshserver:
+            # ssh, write new connection file
+            self.kernel_manager.write_connection_file()
+        atexit.register(self.kernel_manager.cleanup_connection_file)
+        self.kernel_manager.start_channels()
+
+
+    def initialize(self, argv=None):
+        """
+        Classes which mix this class in should call:
+               IPythonConsoleApp.initialize(self,argv)
+        """
+        self.init_connection_file()
+        default_secure(self.config)
+        self.init_ssh()
+        self.init_kernel_manager()
+

IPython/frontend/terminal/console/app.py

@@ -0,0 +1,152
+""" A minimal application using the ZMQ-based terminal IPython frontend.
+
+This is not a complete console app, as subprocess will not be able to receive
+input, there is no real readline support, among other limitations.
+
+Authors:
+
+* Min RK
+* Paul Ivanov
+
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import signal
+import sys
+import time
+
+from IPython.frontend.terminal.ipapp import TerminalIPythonApp, frontend_flags as term_flags
+
+from IPython.utils.traitlets import (
+    Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
+)
+from IPython.utils.warn import warn,error
+
+from IPython.zmq.ipkernel import IPKernelApp
+from IPython.zmq.session import Session, default_secure
+from IPython.zmq.zmqshell import ZMQInteractiveShell
+from IPython.frontend.consoleapp import (
+        IPythonConsoleApp, app_aliases, app_flags, aliases, app_aliases, flags
+    )
+
+from IPython.frontend.terminal.console.interactiveshell import ZMQTerminalInteractiveShell
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+_examples = """
+ipython console # start the ZMQ-based console
+ipython console --existing # connect to an existing ipython session
+"""
+
+#-----------------------------------------------------------------------------
+# Flags and Aliases
+#-----------------------------------------------------------------------------
+
+# copy flags from mixin:
+flags = dict(flags)
+# start with mixin frontend flags:
+frontend_flags = dict(app_flags)
+# add TerminalIPApp flags:
+frontend_flags.update(term_flags)
+# pylab is not frontend-specific in two-process IPython
+frontend_flags.pop('pylab')
+# disable quick startup, as it won't propagate to the kernel anyway
+frontend_flags.pop('quick')
+# update full dict with frontend flags:
+flags.update(frontend_flags)
+
+# copy flags from mixin
+aliases = dict(aliases)
+# start with mixin frontend flags
+frontend_aliases = dict(app_aliases)
+# load updated frontend flags into full dict
+aliases.update(frontend_aliases)
+
+# get flags&aliases into sets, and remove a couple that
+# shouldn't be scrubbed from backend flags:
+frontend_aliases = set(frontend_aliases.keys())
+frontend_flags = set(frontend_flags.keys())
+
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class ZMQTerminalIPythonApp(TerminalIPythonApp, IPythonConsoleApp):
+    name = "ipython-console"
+    """Start a terminal frontend to the IPython zmq kernel."""
+
+    description = """
+        The IPython terminal-based Console.
+
+        This launches a Console application inside a terminal.
+
+        The Console supports various extra features beyond the traditional
+        single-process Terminal IPython shell, such as connecting to an
+        existing ipython session, via:
+
+            ipython console --existing
+
+        where the previous session could have been created by another ipython
+        console, an ipython qtconsole, or by opening an ipython notebook.
+
+    """
+    examples = _examples
+
+    classes = List([IPKernelApp, ZMQTerminalInteractiveShell, Session])
+    flags = Dict(flags)
+    aliases = Dict(aliases)
+    frontend_aliases = Any(frontend_aliases)
+    frontend_flags = Any(frontend_flags)
+    
+    subcommands = Dict()
+
+    def parse_command_line(self, argv=None):
+        super(ZMQTerminalIPythonApp, self).parse_command_line(argv)
+        self.build_kernel_argv(argv)
+
+    def init_shell(self):
+        IPythonConsoleApp.initialize(self)
+        # relay sigint to kernel
+        signal.signal(signal.SIGINT, self.handle_sigint)
+        self.shell = ZMQTerminalInteractiveShell.instance(config=self.config,
+                        display_banner=False, profile_dir=self.profile_dir,
+                        ipython_dir=self.ipython_dir, kernel_manager=self.kernel_manager)
+
+    def handle_sigint(self, *args):
+        if self.shell._executing:
+            if self.kernel_manager.has_kernel:
+                # interrupt already gets passed to subprocess by signal handler.
+                # Only if we prevent that should we need to explicitly call
+                # interrupt_kernel, until which time, this would result in a 
+                # double-interrupt:
+                # self.kernel_manager.interrupt_kernel()
+                pass
+            else:
+                self.shell.write_err('\n')
+                error("Cannot interrupt kernels we didn't start.\n")
+        else:
+            # raise the KeyboardInterrupt if we aren't waiting for execution,
+            # so that the interact loop advances, and prompt is redrawn, etc.
+            raise KeyboardInterrupt
+            
+
+    def init_code(self):
+        # no-op in the frontend, code gets run in the backend
+        pass
+
+def launch_new_instance():
+    """Create and run a full blown IPython instance"""
+    app = ZMQTerminalIPythonApp.instance()
+    app.initialize()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/frontend/terminal/console/completer.py

@@ -0,0 +1,44
+# -*- coding: utf-8 -*-
+import readline
+from Queue import Empty
+
+class ZMQCompleter(object):
+    """Client-side completion machinery.
+
+    How it works: self.complete will be called multiple times, with
+    state=0,1,2,... When state=0 it should compute ALL the completion matches,
+    and then return them for each value of state."""
+    
+    def __init__(self, shell, km):
+        self.shell = shell
+        self.km =  km
+        self.matches = []
+        
+    def complete_request(self,text):
+        line = readline.get_line_buffer()
+        cursor_pos = readline.get_endidx()
+        
+        # send completion request to kernel
+        # Give the kernel up to 0.5s to respond
+        msg_id = self.km.shell_channel.complete(text=text, line=line,
+                                                        cursor_pos=cursor_pos)
+        
+        msg = self.km.shell_channel.get_msg(timeout=0.5)
+        if msg['parent_header']['msg_id'] == msg_id:
+            return msg["content"]["matches"]
+        return []
+    
+    def rlcomplete(self, text, state):
+        if state == 0:
+            try:
+                self.matches = self.complete_request(text)
+            except Empty:
+                print('WARNING: Kernel timeout on tab completion.')
+        
+        try:
+            return self.matches[state]
+        except IndexError:
+            return None
+    
+    def complete(self, text, line, cursor_pos=None):
+        return self.rlcomplete(text, 0)

IPython/frontend/terminal/console/interactiveshell.py

@@ -0,0 +1,342
+# -*- coding: utf-8 -*-
+"""Frontend of ipython working with python-zmq
+
+Ipython's frontend, is a ipython interface that send request to kernel and proccess the kernel's outputs.
+
+For more details, see the ipython-zmq design
+"""
+#-----------------------------------------------------------------------------
+# Copyright (C) 2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+import bdb
+import signal
+import sys
+import time
+
+from Queue import Empty
+
+from IPython.core.alias import AliasManager, AliasError
+from IPython.core import page
+from IPython.utils.warn import warn, error, fatal
+from IPython.utils import io
+
+from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
+from IPython.frontend.terminal.console.completer import ZMQCompleter
+
+
+class ZMQTerminalInteractiveShell(TerminalInteractiveShell):
+    """A subclass of TerminalInteractiveShell that uses the 0MQ kernel"""
+    _executing = False
+    
+    def __init__(self, *args, **kwargs):
+        self.km = kwargs.pop('kernel_manager')
+        self.session_id = self.km.session.session
+        super(ZMQTerminalInteractiveShell, self).__init__(*args, **kwargs)
+        
+    def init_completer(self):
+        """Initialize the completion machinery.
+
+        This creates completion machinery that can be used by client code,
+        either interactively in-process (typically triggered by the readline
+        library), programatically (such as in test suites) or out-of-prcess
+        (typically over the network by remote frontends).
+        """
+        from IPython.core.completerlib import (module_completer,
+                                               magic_run_completer, cd_completer)
+        
+        self.Completer = ZMQCompleter(self, self.km)
+        
+
+        self.set_hook('complete_command', module_completer, str_key = 'import')
+        self.set_hook('complete_command', module_completer, str_key = 'from')
+        self.set_hook('complete_command', magic_run_completer, str_key = '%run')
+        self.set_hook('complete_command', cd_completer, str_key = '%cd')
+
+        # Only configure readline if we truly are using readline.  IPython can
+        # do tab-completion over the network, in GUIs, etc, where readline
+        # itself may be absent
+        if self.has_readline:
+            self.set_readline_completer()
+    
+    def run_cell(self, cell, store_history=True):
+        """Run a complete IPython cell.
+        
+        Parameters
+        ----------
+        cell : str
+          The code (including IPython code such as %magic functions) to run.
+        store_history : bool
+          If True, the raw and translated cell will be stored in IPython's
+          history. For user code calling back into IPython's machinery, this
+          should be set to False.
+        """
+        if (not cell) or cell.isspace():
+            return
+
+        if cell.strip() == 'exit':
+            # explicitly handle 'exit' command
+            return self.ask_exit()
+
+        self._executing = True
+        # flush stale replies, which could have been ignored, due to missed heartbeats
+        while self.km.shell_channel.msg_ready():
+            self.km.shell_channel.get_msg()
+        # shell_channel.execute takes 'hidden', which is the inverse of store_hist
+        msg_id = self.km.shell_channel.execute(cell, not store_history)
+        while not self.km.shell_channel.msg_ready() and self.km.is_alive:
+            try:
+                self.handle_stdin_request(timeout=0.05)
+            except Empty:
+                # display intermediate print statements, etc.
+                self.handle_iopub()
+                pass
+        if self.km.shell_channel.msg_ready():
+            self.handle_execute_reply(msg_id)
+        self._executing = False
+
+    #-----------------
+    # message handlers
+    #-----------------
+
+    def handle_execute_reply(self, msg_id):
+        msg = self.km.shell_channel.get_msg()
+        if msg["parent_header"].get("msg_id", None) == msg_id:
+            
+            self.handle_iopub()
+            
+            content = msg["content"]
+            status = content['status']
+            
+            if status == 'aborted':
+                self.write('Aborted\n')
+                return
+            elif status == 'ok':
+                # print execution payloads as well:
+                for item in content["payload"]:
+                    text = item.get('text', None)
+                    if text:
+                        page.page(text)
+               
+            elif status == 'error':
+                for frame in content["traceback"]:
+                    print(frame, file=io.stderr)
+            
+            self.execution_count = int(content["execution_count"] + 1)
+
+
+    def handle_iopub(self):
+        """ Method to procces subscribe channel's messages
+
+           This method reads a message and processes the content in different
+           outputs like stdout, stderr, pyout and status
+
+           Arguments:
+           sub_msg:  message receive from kernel in the sub socket channel
+                     capture by kernel manager.
+        """
+        while self.km.sub_channel.msg_ready():
+            sub_msg = self.km.sub_channel.get_msg()
+            msg_type = sub_msg['header']['msg_type']
+            parent = sub_msg["parent_header"]
+            if (not parent) or self.session_id == parent['session']:
+                if msg_type == 'status' :
+                    if sub_msg["content"]["execution_state"] == "busy" :
+                        pass
+
+                elif msg_type == 'stream' :
+                    if sub_msg["content"]["name"] == "stdout":
+                        print(sub_msg["content"]["data"], file=io.stdout, end="")
+                        io.stdout.flush()
+                    elif sub_msg["content"]["name"] == "stderr" :
+                        print(sub_msg["content"]["data"], file=io.stderr, end="")
+                        io.stderr.flush()
+
+                elif msg_type == 'pyout':
+                    self.execution_count = int(sub_msg["content"]["execution_count"])
+                    format_dict = sub_msg["content"]["data"]
+                    # taken from DisplayHook.__call__:
+                    hook = self.displayhook
+                    hook.start_displayhook()
+                    hook.write_output_prompt()
+                    hook.write_format_data(format_dict)
+                    hook.log_output(format_dict)
+                    hook.finish_displayhook()
+
+    def handle_stdin_request(self, timeout=0.1):
+        """ Method to capture raw_input
+        """
+        msg_rep = self.km.stdin_channel.get_msg(timeout=timeout)
+        # in case any iopub came while we were waiting:
+        self.handle_iopub()
+        if self.session_id == msg_rep["parent_header"].get("session"):
+            # wrap SIGINT handler
+            real_handler = signal.getsignal(signal.SIGINT)
+            def double_int(sig,frame):
+                # call real handler (forwards sigint to kernel),
+                # then raise local interrupt, stopping local raw_input
+                real_handler(sig,frame)
+                raise KeyboardInterrupt
+            signal.signal(signal.SIGINT, double_int)
+            
+            try:
+                raw_data = raw_input(msg_rep["content"]["prompt"])
+            except EOFError:
+                # turn EOFError into EOF character
+                raw_data = '\x04'
+            except KeyboardInterrupt:
+                sys.stdout.write('\n')
+                return
+            finally:
+                # restore SIGINT handler
+                signal.signal(signal.SIGINT, real_handler)
+            
+            # only send stdin reply if there *was not* another request
+            # or execution finished while we were reading.
+            if not (self.km.stdin_channel.msg_ready() or self.km.shell_channel.msg_ready()):
+                self.km.stdin_channel.input(raw_data)
+
+    def mainloop(self, display_banner=False):
+        while True:
+            try:
+                self.interact(display_banner=display_banner)
+                #self.interact_with_readline()                
+                # XXX for testing of a readline-decoupled repl loop, call
+                # interact_with_readline above
+                break
+            except KeyboardInterrupt:
+                # this should not be necessary, but KeyboardInterrupt
+                # handling seems rather unpredictable...
+                self.write("\nKeyboardInterrupt in interact()\n")
+    
+    def wait_for_kernel(self, timeout=None):
+        """method to wait for a kernel to be ready"""
+        tic = time.time()
+        self.km.hb_channel.unpause()
+        while True:
+            self.run_cell('1', False)
+            if self.km.hb_channel.is_beating():
+                # heart failure was not the reason this returned
+                break
+            else:
+                # heart failed
+                if timeout is not None and (time.time() - tic) > timeout:
+                    return False
+        return True
+    
+    def interact(self, display_banner=None):
+        """Closely emulate the interactive Python console."""
+
+        # batch run -> do not interact
+        if self.exit_now:
+            return
+
+        if display_banner is None:
+            display_banner = self.display_banner
+        
+        if isinstance(display_banner, basestring):
+            self.show_banner(display_banner)
+        elif display_banner:
+            self.show_banner()
+
+        more = False
+        
+        # run a non-empty no-op, so that we don't get a prompt until
+        # we know the kernel is ready. This keeps the connection
+        # message above the first prompt.
+        if not self.wait_for_kernel(3):
+            error("Kernel did not respond\n")
+            return
+        
+        if self.has_readline:
+            self.readline_startup_hook(self.pre_readline)
+            hlen_b4_cell = self.readline.get_current_history_length()
+        else:
+            hlen_b4_cell = 0
+        # exit_now is set by a call to %Exit or %Quit, through the
+        # ask_exit callback.
+
+        while not self.exit_now:
+            if not self.km.is_alive:
+                # kernel died, prompt for action or exit
+                action = "restart" if self.km.has_kernel else "wait for restart"
+                ans = self.ask_yes_no("kernel died, %s ([y]/n)?" % action, default='y')
+                if ans:
+                    if self.km.has_kernel:
+                        self.km.restart_kernel(True)
+                    self.wait_for_kernel(3)
+                else:
+                    self.exit_now = True
+                continue
+            try:
+                # protect prompt block from KeyboardInterrupt
+                # when sitting on ctrl-C
+                self.hooks.pre_prompt_hook()
+                if more:
+                    try:
+                        prompt = self.prompt_manager.render('in2')
+                    except Exception:
+                        self.showtraceback()
+                    if self.autoindent:
+                        self.rl_do_indent = True
+                    
+                else:
+                    try:
+                        prompt = self.separate_in + self.prompt_manager.render('in')
+                    except Exception:
+                        self.showtraceback()
+                
+                line = self.raw_input(prompt)
+                if self.exit_now:
+                    # quick exit on sys.std[in|out] close
+                    break
+                if self.autoindent:
+                    self.rl_do_indent = False
+                    
+            except KeyboardInterrupt:
+                #double-guard against keyboardinterrupts during kbdint handling
+                try:
+                    self.write('\nKeyboardInterrupt\n')
+                    source_raw = self.input_splitter.source_raw_reset()[1]
+                    hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
+                    more = False
+                except KeyboardInterrupt:
+                    pass
+            except EOFError:
+                if self.autoindent:
+                    self.rl_do_indent = False
+                    if self.has_readline:
+                        self.readline_startup_hook(None)
+                self.write('\n')
+                self.exit()
+            except bdb.BdbQuit:
+                warn('The Python debugger has exited with a BdbQuit exception.\n'
+                     'Because of how pdb handles the stack, it is impossible\n'
+                     'for IPython to properly format this particular exception.\n'
+                     'IPython will resume normal operation.')
+            except:
+                # exceptions here are VERY RARE, but they can be triggered
+                # asynchronously by signal handlers, for example.
+                self.showtraceback()
+            else:
+                self.input_splitter.push(line)
+                more = self.input_splitter.push_accepts_more()
+                if (self.SyntaxTB.last_syntax_error and
+                    self.autoedit_syntax):
+                    self.edit_syntax_error()
+                if not more:
+                    source_raw = self.input_splitter.source_reset()
+                    hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
+                    self.run_cell(source_raw)
+                
+
+        # Turn off the exit flag, so the mainloop can be restarted if desired
+        self.exit_now = False

IPython/frontend/terminal/console/tests/test_console.py

@@ -0,0 +1,59
+"""Tests for two-process terminal frontend
+
+Currenlty only has the most simple test possible, starting a console and running
+a single command.
+
+Authors:
+
+* Min RK
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import time
+
+import nose.tools as nt
+from nose import SkipTest
+
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+from IPython.utils import py3compat
+from IPython.utils.process import find_cmd
+
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
[e-mail]_win32
+def test_console_starts():
+    """test that `ipython console` starts a terminal"""
+    from IPython.external import pexpect
+    
+    # weird IOErrors prevent this from firing sometimes:
+    ipython_cmd = None
+    for i in range(5):
+        try:
+            ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython')
+        except IOError:
+            time.sleep(0.1)
+        else:
+            break
+    if ipython_cmd is None:
+        raise SkipTest("Could not determine ipython command")
+    
+    p = pexpect.spawn(ipython_cmd, args=['console', '--colors=NoColor'])
+    idx = p.expect([r'In \[\d+\]', pexpect.EOF], timeout=4)
+    nt.assert_equals(idx, 0, "expected in prompt")
+    p.sendline('5')
+    idx = p.expect([r'Out\[\d+\]: 5', pexpect.EOF], timeout=1)
+    nt.assert_equals(idx, 0, "expected out prompt")
+    idx = p.expect([r'In \[\d+\]', pexpect.EOF], timeout=1)
+    nt.assert_equals(idx, 0, "expected second in prompt")
+    # send ctrl-D;ctrl-D to exit
+    p.sendeof()
+    p.sendeof()
+    p.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=1)
+    if p.isalive():
+        p.terminate()

IPython/lib/tests/test_kernel.py

@@ -0,0 +1,63
+"""Tests for kernel utility functions
+
+Authors
+-------
+* MinRK
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2011, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib imports
+from unittest import TestCase
+
+# Third-party imports
+import nose.tools as nt
+
+# Our own imports
+from IPython.testing import decorators as dec
+from IPython.lib import kernel
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
[e-mail]
+def test_swallow_argv():
+    tests = [
+        # expected  , argv       , aliases, flags
+        (['-a', '5'], ['-a', '5'], None, None),
+        (['5'], ['-a', '5'], None, ['a']),
+        ([], ['-a', '5'], ['a'], None),
+        ([], ['-a', '5'], ['a'], ['a']),
+        ([], ['--foo'], None, ['foo']),
+        (['--foo'], ['--foo'], ['foobar'], []),
+        ([], ['--foo', '5'], ['foo'], []),
+        ([], ['--foo=5'], ['foo'], []),
+        (['--foo=5'], ['--foo=5'], [], ['foo']),
+        (['5'], ['--foo', '5'], [], ['foo']),
+        (['bar'], ['--foo', '5', 'bar'], ['foo'], ['foo']),
+        (['bar'], ['--foo=5', 'bar'], ['foo'], ['foo']),
+        (['5','bar'], ['--foo', '5', 'bar'], None, ['foo']),
+        (['bar'], ['--foo', '5', 'bar'], ['foo'], None),
+        (['bar'], ['--foo=5', 'bar'], ['foo'], None),
+    ]
+    for expected, argv, aliases, flags in tests:
+        stripped = kernel.swallow_argv(argv, aliases=aliases, flags=flags)
+        message = '\n'.join(['',
+            "argv: %r" % argv,
+            "aliases: %r" % aliases,
+            "flags : %r" % flags,
+            "expected : %r" % expected,
+            "returned : %r" % stripped,
+        ])
+        yield nt.assert_equal(expected, stripped, message)
+

IPython/frontend/html/notebook/notebookapp.py

@@ -1,385 +1,369
 """A tornado based IPython notebook server.
 
 Authors:
 
 * Brian Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import errno
 import logging
 import os
 import signal
 import socket
 import sys
 import threading
 import webbrowser
 
 # Third party
 import zmq
 
 # Install the pyzmq ioloop. This has to be done before anything else from
 # tornado is imported.
 from zmq.eventloop import ioloop
 # FIXME: ioloop.install is new in pyzmq-2.1.7, so remove this conditional
 # when pyzmq dependency is updated beyond that.
 if hasattr(ioloop, 'install'):
     ioloop.install()
 else:
     import tornado.ioloop
     tornado.ioloop.IOLoop = ioloop.IOLoop
 
 from tornado import httpserver
 from tornado import web
 
 # Our own libraries
 from .kernelmanager import MappingKernelManager
 from .handlers import (LoginHandler, LogoutHandler,
     ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
     MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
     ShellHandler, NotebookRootHandler, NotebookHandler, RSTHandler
 )
 from .notebookmanager import NotebookManager
 
 from IPython.config.application import catch_config_error
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir
+from IPython.lib.kernel import swallow_argv
 from IPython.zmq.session import Session, default_secure
 from IPython.zmq.zmqshell import ZMQInteractiveShell
 from IPython.zmq.ipkernel import (
     flags as ipkernel_flags,
     aliases as ipkernel_aliases,
     IPKernelApp
 )
 from IPython.utils.traitlets import Dict, Unicode, Integer, List, Enum, Bool
 
 #-----------------------------------------------------------------------------
 # Module globals
 #-----------------------------------------------------------------------------
 
 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
 _kernel_action_regex = r"(?P<action>restart|interrupt)"
 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
 
 LOCALHOST = '127.0.0.1'
 
 _examples = """
 ipython notebook                       # start the notebook
 ipython notebook --profile=sympy       # use the sympy profile
 ipython notebook --pylab=inline        # pylab in inline plotting mode
 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
 ipython notebook --port=5555 --ip=*    # Listen on port 5555, all interfaces
 """
 
 #-----------------------------------------------------------------------------
 # The Tornado web application
 #-----------------------------------------------------------------------------
 
 class NotebookWebApplication(web.Application):
 
     def __init__(self, ipython_app, kernel_manager, notebook_manager, log):
         handlers = [
             (r"/", ProjectDashboardHandler),
             (r"/login", LoginHandler),
             (r"/logout", LogoutHandler),
             (r"/new", NewHandler),
             (r"/%s" % _notebook_id_regex, NamedNotebookHandler),
             (r"/kernels", MainKernelHandler),
             (r"/kernels/%s" % _kernel_id_regex, KernelHandler),
             (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
             (r"/kernels/%s/iopub" % _kernel_id_regex, IOPubHandler),
             (r"/kernels/%s/shell" % _kernel_id_regex, ShellHandler),
             (r"/notebooks", NotebookRootHandler),
             (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
             (r"/rstservice/render", RSTHandler)
         ]
         settings = dict(
             template_path=os.path.join(os.path.dirname(__file__), "templates"),
             static_path=os.path.join(os.path.dirname(__file__), "static"),
             cookie_secret=os.urandom(1024),
             login_url="/login",
         )
         web.Application.__init__(self, handlers, **settings)
 
         self.kernel_manager = kernel_manager
         self.log = log
         self.notebook_manager = notebook_manager
         self.ipython_app = ipython_app
         self.read_only = self.ipython_app.read_only
 
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = dict(ipkernel_flags)
 flags['no-browser']=(
     {'NotebookApp' : {'open_browser' : False}},
     "Don't open the notebook in a browser after startup."
 )
 flags['no-mathjax']=(
     {'NotebookApp' : {'enable_mathjax' : False}},
     """Disable MathJax
     
     MathJax is the javascript library IPython uses to render math/LaTeX. It is
     very large, so you may want to disable it if you have a slow internet
     connection, or for offline use of the notebook.
     
     When disabled, equations etc. will appear as their untransformed TeX source.
     """
 )
 flags['read-only'] = (
     {'NotebookApp' : {'read_only' : True}},
     """Allow read-only access to notebooks.
     
     When using a password to protect the notebook server, this flag
     allows unauthenticated clients to view the notebook list, and
     individual notebooks, but not edit them, start kernels, or run
     code.
     
     If no password is set, the server will be entirely read-only.
     """
 )
 
 # the flags that are specific to the frontend
 # these must be scrubbed before being passed to the kernel,
 # or it will raise an error on unrecognized flags
 notebook_flags = ['no-browser', 'no-mathjax', 'read-only']
 
 aliases = dict(ipkernel_aliases)
 
 aliases.update({
     'ip': 'NotebookApp.ip',
     'port': 'NotebookApp.port',
     'keyfile': 'NotebookApp.keyfile',
     'certfile': 'NotebookApp.certfile',
     'notebook-dir': 'NotebookManager.notebook_dir',
 })
 
 # remove ipkernel flags that are singletons, and don't make sense in
 # multi-kernel evironment:
 aliases.pop('f', None)
 
 notebook_aliases = [u'port', u'ip', u'keyfile', u'certfile',
                     u'notebook-dir']
 
 #-----------------------------------------------------------------------------
 # NotebookApp
 #-----------------------------------------------------------------------------
 
 class NotebookApp(BaseIPythonApplication):
 
     name = 'ipython-notebook'
     default_config_file_name='ipython_notebook_config.py'
     
     description = """
         The IPython HTML Notebook.
         
         This launches a Tornado based HTML Notebook Server that serves up an
         HTML5/Javascript Notebook client.
     """
     examples = _examples
     
     classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session,
                MappingKernelManager, NotebookManager]
     flags = Dict(flags)
     aliases = Dict(aliases)
 
     kernel_argv = List(Unicode)
 
     log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
                     default_value=logging.INFO,
                     config=True,
                     help="Set the log level by value or name.")
 
     # Network related information.
 
     ip = Unicode(LOCALHOST, config=True,
         help="The IP address the notebook server will listen on."
     )
 
     def _ip_changed(self, name, old, new):
         if new == u'*': self.ip = u''
 
     port = Integer(8888, config=True,
         help="The port the notebook server will listen on."
     )
 
     certfile = Unicode(u'', config=True, 
         help="""The full path to an SSL/TLS certificate file."""
     )
     
     keyfile = Unicode(u'', config=True, 
         help="""The full path to a private key file for usage with SSL/TLS."""
     )
 
     password = Unicode(u'', config=True,
                       help="""Hashed password to use for web authentication.
 
                       To generate, type in a python/IPython shell:
 
                         from IPython.lib import passwd; passwd()
 
                       The string should be of the form type:salt:hashed-password.
                       """
     )
     
     open_browser = Bool(True, config=True,
                         help="Whether to open in a browser after starting.")
     
     read_only = Bool(False, config=True,
         help="Whether to prevent editing/execution of notebooks."
     )
     
     enable_mathjax = Bool(True, config=True,
         help="""Whether to enable MathJax for typesetting math/TeX
 
         MathJax is the javascript library IPython uses to render math/LaTeX. It is
         very large, so you may want to disable it if you have a slow internet
         connection, or for offline use of the notebook.
 
         When disabled, equations etc. will appear as their untransformed TeX source.
         """
     )
     def _enable_mathjax_changed(self, name, old, new):
         """set mathjax url to empty if mathjax is disabled"""
         if not new:
             self.mathjax_url = u''
     
     mathjax_url = Unicode("", config=True,
         help="""The url for MathJax.js."""
     )
     def _mathjax_url_default(self):
         if not self.enable_mathjax:
             return u''
         static_path = os.path.join(os.path.dirname(__file__), "static")
         if os.path.exists(os.path.join(static_path, 'mathjax', "MathJax.js")):
             self.log.info("Using local MathJax")
             return u"static/mathjax/MathJax.js"
         else:
             self.log.info("Using MathJax from CDN")
             return u"http://cdn.mathjax.org/mathjax/latest/MathJax.js"
     
     def _mathjax_url_changed(self, name, old, new):
         if new and not self.enable_mathjax:
             # enable_mathjax=False overrides mathjax_url
             self.mathjax_url = u''
         else:
             self.log.info("Using MathJax: %s", new)
 
     def parse_command_line(self, argv=None):
         super(NotebookApp, self).parse_command_line(argv)
         if argv is None:
             argv = sys.argv[1:]
 
-        self.kernel_argv = list(argv) # copy
+        # Scrub frontend-specific flags
+        self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
         # Kernel should inherit default config file from frontend
         self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
-        # Scrub frontend-specific flags
-        for a in argv:
-            if a.startswith('-') and a.lstrip('-') in notebook_flags:
-                self.kernel_argv.remove(a)
-        swallow_next = False
-        for a in argv:
-            if swallow_next:
-                self.kernel_argv.remove(a)
-                swallow_next = False
-                continue
-            if a.startswith('-'):
-                split = a.lstrip('-').split('=')
-                alias = split[0]
-                if alias in notebook_aliases:
-                    self.kernel_argv.remove(a)
-                    if len(split) == 1:
-                        # alias passed with arg via space
-                        swallow_next = True
 
     def init_configurables(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
         # force Session default to be secure
         default_secure(self.config)
         # Create a KernelManager and start a kernel.
         self.kernel_manager = MappingKernelManager(
             config=self.config, log=self.log, kernel_argv=self.kernel_argv,
             connection_dir = self.profile_dir.security_dir,
         )
         self.notebook_manager = NotebookManager(config=self.config, log=self.log)
         self.notebook_manager.list_notebooks()
 
     def init_logging(self):
         super(NotebookApp, self).init_logging()
         # This prevents double log messages because tornado use a root logger that
         # self.log is a child of. The logging module dipatches log messages to a log
         # and all of its ancenstors until propagate is set to False.
         self.log.propagate = False
 
     @catch_config_error
     def initialize(self, argv=None):
         super(NotebookApp, self).initialize(argv)
         self.init_configurables()
         self.web_app = NotebookWebApplication(
             self, self.kernel_manager, self.notebook_manager, self.log
         )
         if self.certfile:
             ssl_options = dict(certfile=self.certfile)
             if self.keyfile:
                 ssl_options['keyfile'] = self.keyfile
         else:
             ssl_options = None
         self.web_app.password = self.password
         self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
         if ssl_options is None and not self.ip:
             self.log.critical('WARNING: the notebook server is listening on all IP addresses '
                               'but not using any encryption or authentication. This is highly '
                               'insecure and not recommended.')
 
         # Try random ports centered around the default.
         from random import randint
         n = 50  # Max number of attempts, keep reasonably large.
         for port in range(self.port, self.port+5) + [self.port + randint(-2*n, 2*n) for i in range(n-5)]:
             try:
                 self.http_server.listen(port, self.ip)
             except socket.error, e:
                 if e.errno != errno.EADDRINUSE:
                     raise
                 self.log.info('The port %i is already in use, trying another random port.' % port)
             else:
                 self.port = port
                 break
 
     def start(self):
         ip = self.ip if self.ip else '[all ip addresses on your system]'
         proto = 'https' if self.certfile else 'http'
         self.log.info("The IPython Notebook is running at: %s://%s:%i" % (proto,
                                                                           ip,
                                                                           self.port))
         if self.open_browser:
             ip = self.ip or '127.0.0.1'
             b = lambda : webbrowser.open("%s://%s:%i" % (proto, ip, self.port),
                                          new=2)
             threading.Thread(target=b).start()
 
         ioloop.IOLoop.instance().start()
 
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 def launch_new_instance():
     app = NotebookApp()
     app.initialize()
     app.start()
 

IPython/frontend/html/notebook/tests/test_kernelsession.py

@@ -1,26 +1,27
 """Tests for the notebook kernel and session manager."""
 
 from unittest import TestCase
 
 from IPython.frontend.html.notebook.kernelmanager import MultiKernelManager
 
 class TestKernelManager(TestCase):
 
     def test_km_lifecycle(self):
         km = MultiKernelManager()
         kid = km.start_kernel()
         self.assert_(kid in km)
         self.assertEquals(len(km),1)
         km.kill_kernel(kid)
         self.assert_(not kid in km)
 
         kid = km.start_kernel()
         self.assertEquals('127.0.0.1',km.get_kernel_ip(kid))
         port_dict = km.get_kernel_ports(kid)
         self.assert_('stdin_port' in port_dict)
         self.assert_('iopub_port' in port_dict)
         self.assert_('shell_port' in port_dict)
         self.assert_('hb_port' in port_dict)
         km.get_kernel(kid)
+        km.kill_kernel(kid)
 
 

IPython/frontend/qt/console/qtconsoleapp.py

@@ -1,549 +1,352
 """ A minimal application using the Qt console-style IPython frontend.
 
 This is not a complete console app, as subprocess will not be able to receive
 input, there is no real readline support, among other limitations.
 
 Authors:
 
 * Evan Patterson
 * Min RK
 * Erik Tollerud
 * Fernando Perez
 * Bussonnier Matthias
 * Thomas Kluyver
+* Paul Ivanov
 
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # stdlib imports
 import json
 import os
 import signal
 import sys
 import uuid
 
 # System library imports
-from IPython.external.qt import QtGui
+from IPython.external.qt import QtCore, QtGui
 
 # Local imports
 from IPython.config.application import boolean_flag, catch_config_error
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir
 from IPython.lib.kernel import tunnel_to_kernel, find_connection_file
 from IPython.frontend.qt.console.frontend_widget import FrontendWidget
 from IPython.frontend.qt.console.ipython_widget import IPythonWidget
 from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget
 from IPython.frontend.qt.console import styles
 from IPython.frontend.qt.console.mainwindow import MainWindow
 from IPython.frontend.qt.kernelmanager import QtKernelManager
 from IPython.utils.path import filefind
 from IPython.utils.py3compat import str_to_bytes
 from IPython.utils.traitlets import (
     Dict, List, Unicode, Integer, CaselessStrEnum, CBool, Any
 )
-from IPython.zmq.ipkernel import (
-    flags as ipkernel_flags,
-    aliases as ipkernel_aliases,
-    IPKernelApp
-)
+from IPython.zmq.ipkernel import IPKernelApp
 from IPython.zmq.session import Session, default_secure
 from IPython.zmq.zmqshell import ZMQInteractiveShell
 
+from IPython.frontend.consoleapp import (
+        IPythonConsoleApp, app_aliases, app_flags, flags, aliases
+    )
+
 #-----------------------------------------------------------------------------
 # Network Constants
 #-----------------------------------------------------------------------------
 
 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 _examples = """
 ipython qtconsole                 # start the qtconsole
 ipython qtconsole --pylab=inline  # start with pylab in inline plotting mode
 """
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
-flags = dict(ipkernel_flags)
+# start with copy of flags
+flags = dict(flags)
 qt_flags = {
-    'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
-            "Connect to an existing kernel. If no argument specified, guess most recent"),
     'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
             "Use a pure Python kernel instead of an IPython kernel."),
     'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
             "Disable rich text support."),
 }
 qt_flags.update(boolean_flag(
     'gui-completion', 'ConsoleWidget.gui_completion',
     "use a GUI widget for tab completion",
     "use plaintext output for completion"
 ))
-qt_flags.update(boolean_flag(
-    'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
-    """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
-       to force a direct exit without any confirmation.
-    """,
-    """Don't prompt the user when exiting. This will terminate the kernel
-       if it is owned by the frontend, and leave it alive if it is external.
-    """
-))
+# and app_flags from the Console Mixin
+qt_flags.update(app_flags)
+# add frontend flags to the full set
 flags.update(qt_flags)
 
-aliases = dict(ipkernel_aliases)
-
+# start with copy of front&backend aliases list
+aliases = dict(aliases)
 qt_aliases = dict(
-    hb = 'IPythonQtConsoleApp.hb_port',
-    shell = 'IPythonQtConsoleApp.shell_port',
-    iopub = 'IPythonQtConsoleApp.iopub_port',
-    stdin = 'IPythonQtConsoleApp.stdin_port',
-    ip = 'IPythonQtConsoleApp.ip',
-    existing = 'IPythonQtConsoleApp.existing',
-    f = 'IPythonQtConsoleApp.connection_file',
 
     style = 'IPythonWidget.syntax_style',
     stylesheet = 'IPythonQtConsoleApp.stylesheet',
     colors = 'ZMQInteractiveShell.colors',
 
     editor = 'IPythonWidget.editor',
     paging = 'ConsoleWidget.paging',
-    ssh = 'IPythonQtConsoleApp.sshserver',
 )
+# and app_aliases from the Console Mixin
+qt_aliases.update(app_aliases)
+# add frontend aliases to the full set
 aliases.update(qt_aliases)
 
+# get flags&aliases into sets, and remove a couple that
+# shouldn't be scrubbed from backend flags:
+qt_aliases = set(qt_aliases.keys())
+qt_aliases.remove('colors')
+qt_flags = set(qt_flags.keys())
+
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # IPythonQtConsole
 #-----------------------------------------------------------------------------
 
 
-class IPythonQtConsoleApp(BaseIPythonApplication):
+class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp):
     name = 'ipython-qtconsole'
-    default_config_file_name='ipython_config.py'
 
     description = """
         The IPython QtConsole.
         
         This launches a Console-style application using Qt.  It is not a full
         console, in that launched terminal subprocesses will not be able to accept
         input.
         
         The QtConsole supports various extra features beyond the Terminal IPython
         shell, such as inline plotting with matplotlib, via:
         
             ipython qtconsole --pylab=inline
         
         as well as saving your session as HTML, and printing the output.
         
     """
     examples = _examples
 
     classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
     flags = Dict(flags)
     aliases = Dict(aliases)
-
-    kernel_argv = List(Unicode)
-
-    # create requested profiles by default, if they don't exist:
-    auto_create = CBool(True)
-    # connection info:
-    ip = Unicode(LOCALHOST, config=True,
-        help="""Set the kernel\'s IP address [default localhost].
-        If the IP address is something other than localhost, then
-        Consoles on other machines will be able to connect
-        to the Kernel, so be careful!"""
-    )
-    
-    sshserver = Unicode('', config=True,
-        help="""The SSH server to use to connect to the kernel.""")
-    sshkey = Unicode('', config=True,
-        help="""Path to the ssh key to use for logging in to the ssh server.""")
-    
-    hb_port = Integer(0, config=True,
-        help="set the heartbeat port [default: random]")
-    shell_port = Integer(0, config=True,
-        help="set the shell (XREP) port [default: random]")
-    iopub_port = Integer(0, config=True,
-        help="set the iopub (PUB) port [default: random]")
-    stdin_port = Integer(0, config=True,
-        help="set the stdin (XREQ) port [default: random]")
-    connection_file = Unicode('', config=True,
-        help="""JSON file in which to store connection info [default: kernel-<pid>.json]
-
-        This file will contain the IP, ports, and authentication key needed to connect
-        clients to this kernel. By default, this file will be created in the security-dir
-        of the current profile, but can be specified by absolute path.
-        """)
-    def _connection_file_default(self):
-        return 'kernel-%i.json' % os.getpid()
-
-    existing = Unicode('', config=True,
-        help="""Connect to an already running kernel""")
+    frontend_flags = Any(qt_flags)
+    frontend_aliases = Any(qt_aliases)
+    kernel_manager_class = QtKernelManager
 
     stylesheet = Unicode('', config=True,
         help="path to a custom CSS stylesheet")
 
-    pure = CBool(False, config=True,
-        help="Use a pure Python kernel instead of an IPython kernel.")
     plain = CBool(False, config=True,
         help="Use a plaintext widget instead of rich text (plain can't print/save).")
 
     def _pure_changed(self, name, old, new):
         kind = 'plain' if self.plain else 'rich'
         self.config.ConsoleWidget.kind = kind
         if self.pure:
             self.widget_factory = FrontendWidget
         elif self.plain:
             self.widget_factory = IPythonWidget
         else:
             self.widget_factory = RichIPythonWidget
 
     _plain_changed = _pure_changed
 
-    confirm_exit = CBool(True, config=True,
-        help="""
-        Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
-        to force a direct exit without any confirmation.""",
-    )
-    
     # the factory for creating a widget
     widget_factory = Any(RichIPythonWidget)
 
     def parse_command_line(self, argv=None):
         super(IPythonQtConsoleApp, self).parse_command_line(argv)
-        if argv is None:
-            argv = sys.argv[1:]
-        self.kernel_argv = list(argv) # copy
-        # kernel should inherit default config file from frontend
-        self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
-        # Scrub frontend-specific flags
-        swallow_next = False
-        was_flag = False
-        # copy again, in case some aliases have the same name as a flag
-        # argv = list(self.kernel_argv)
-        for a in argv:
-            if swallow_next:
-                swallow_next = False
-                # last arg was an alias, remove the next one
-                # *unless* the last alias has a no-arg flag version, in which
-                # case, don't swallow the next arg if it's also a flag:
-                if not (was_flag and a.startswith('-')):
-                    self.kernel_argv.remove(a)
-                    continue
-            if a.startswith('-'):
-                split = a.lstrip('-').split('=')
-                alias = split[0]
-                if alias in qt_aliases:
-                    self.kernel_argv.remove(a)
-                    if len(split) == 1:
-                        # alias passed with arg via space
-                        swallow_next = True
-                        # could have been a flag that matches an alias, e.g. `existing`
-                        # in which case, we might not swallow the next arg
-                        was_flag = alias in qt_flags
-                elif alias in qt_flags:
-                    # strip flag, but don't swallow next, as flags don't take args
-                    self.kernel_argv.remove(a)
-    
-    def init_connection_file(self):
-        """find the connection file, and load the info if found.
-        
-        The current working directory and the current profile's security
-        directory will be searched for the file if it is not given by
-        absolute path.
-        
-        When attempting to connect to an existing kernel and the `--existing`
-        argument does not match an existing file, it will be interpreted as a
-        fileglob, and the matching file in the current profile's security dir
-        with the latest access time will be used.
-        """
-        if self.existing:
-            try:
-                cf = find_connection_file(self.existing)
-            except Exception:
-                self.log.critical("Could not find existing kernel connection file %s", self.existing)
-                self.exit(1)
-            self.log.info("Connecting to existing kernel: %s" % cf)
-            self.connection_file = cf
-        # should load_connection_file only be used for existing?
-        # as it is now, this allows reusing ports if an existing
-        # file is requested
-        try:
-            self.load_connection_file()
-        except Exception:
-            self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
-            self.exit(1)
-    
-    def load_connection_file(self):
-        """load ip/port/hmac config from JSON connection file"""
-        # this is identical to KernelApp.load_connection_file
-        # perhaps it can be centralized somewhere?
-        try:
-            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
-        except IOError:
-            self.log.debug("Connection File not found: %s", self.connection_file)
-            return
-        self.log.debug(u"Loading connection file %s", fname)
-        with open(fname) as f:
-            s = f.read()
-        cfg = json.loads(s)
-        if self.ip == LOCALHOST and 'ip' in cfg:
-            # not overridden by config or cl_args
-            self.ip = cfg['ip']
-        for channel in ('hb', 'shell', 'iopub', 'stdin'):
-            name = channel + '_port'
-            if getattr(self, name) == 0 and name in cfg:
-                # not overridden by config or cl_args
-                setattr(self, name, cfg[name])
-        if 'key' in cfg:
-            self.config.Session.key = str_to_bytes(cfg['key'])
-    
-    def init_ssh(self):
-        """set up ssh tunnels, if needed."""
-        if not self.sshserver and not self.sshkey:
-            return
+        self.build_kernel_argv(argv)
         
-        if self.sshkey and not self.sshserver:
-            # specifying just the key implies that we are connecting directly
-            self.sshserver = self.ip
-            self.ip = LOCALHOST
-        
-        # build connection dict for tunnels:
-        info = dict(ip=self.ip,
-                    shell_port=self.shell_port,
-                    iopub_port=self.iopub_port,
-                    stdin_port=self.stdin_port,
-                    hb_port=self.hb_port
-        )
-        
-        self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
-        
-        # tunnels return a new set of ports, which will be on localhost:
-        self.ip = LOCALHOST
-        try:
-            newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
-        except:
-            # even catch KeyboardInterrupt
-            self.log.error("Could not setup tunnels", exc_info=True)
-            self.exit(1)
-        
-        self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
-        
-        cf = self.connection_file
-        base,ext = os.path.splitext(cf)
-        base = os.path.basename(base)
-        self.connection_file = os.path.basename(base)+'-ssh'+ext
-        self.log.critical("To connect another client via this tunnel, use:")
-        self.log.critical("--existing %s" % self.connection_file)
-    
-    def _new_connection_file(self):
-        return os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % uuid.uuid4())
-
-    def init_kernel_manager(self):
-        # Don't let Qt or ZMQ swallow KeyboardInterupts.
-        signal.signal(signal.SIGINT, signal.SIG_DFL)
-        sec = self.profile_dir.security_dir
-        try:
-            cf = filefind(self.connection_file, ['.', sec])
-        except IOError:
-            # file might not exist
-            if self.connection_file == os.path.basename(self.connection_file):
-                # just shortname, put it in security dir
-                cf = os.path.join(sec, self.connection_file)
-            else:
-                cf = self.connection_file
-
-        # Create a KernelManager and start a kernel.
-        self.kernel_manager = QtKernelManager(
-                                ip=self.ip,
-                                shell_port=self.shell_port,
-                                iopub_port=self.iopub_port,
-                                stdin_port=self.stdin_port,
-                                hb_port=self.hb_port,
-                                connection_file=cf,
-                                config=self.config,
-        )
-        # start the kernel
-        if not self.existing:
-            kwargs = dict(ipython=not self.pure)
-            kwargs['extra_arguments'] = self.kernel_argv
-            self.kernel_manager.start_kernel(**kwargs)
-        elif self.sshserver:
-            # ssh, write new connection file
-            self.kernel_manager.write_connection_file()
-        self.kernel_manager.start_channels()
 
     def new_frontend_master(self):
         """ Create and return new frontend attached to new kernel, launched on localhost.
         """
         ip = self.ip if self.ip in LOCAL_IPS else LOCALHOST
         kernel_manager = QtKernelManager(
                                 ip=ip,
                                 connection_file=self._new_connection_file(),
                                 config=self.config,
         )
         # start the kernel
         kwargs = dict(ipython=not self.pure)
         kwargs['extra_arguments'] = self.kernel_argv
         kernel_manager.start_kernel(**kwargs)
         kernel_manager.start_channels()
         widget = self.widget_factory(config=self.config,
                                    local_kernel=True)
         widget.kernel_manager = kernel_manager
         widget._existing = False
         widget._may_close = True
         widget._confirm_exit = self.confirm_exit
         return widget
 
     def new_frontend_slave(self, current_widget):
         """Create and return a new frontend attached to an existing kernel.
         
         Parameters
         ----------
         current_widget : IPythonWidget
             The IPythonWidget whose kernel this frontend is to share
         """
         kernel_manager = QtKernelManager(
                                 connection_file=current_widget.kernel_manager.connection_file,
                                 config = self.config,
         )
         kernel_manager.load_connection_file()
         kernel_manager.start_channels()
         widget = self.widget_factory(config=self.config,
                                 local_kernel=False)
         widget._existing = True
         widget._may_close = False
         widget._confirm_exit = False
         widget.kernel_manager = kernel_manager
         return widget
 
     def init_qt_elements(self):
         # Create the widget.
         self.app = QtGui.QApplication([])
 
         base_path = os.path.abspath(os.path.dirname(__file__))
         icon_path = os.path.join(base_path, 'resources', 'icon', 'IPythonConsole.svg')
         self.app.icon = QtGui.QIcon(icon_path)
         QtGui.QApplication.setWindowIcon(self.app.icon)
 
         local_kernel = (not self.existing) or self.ip in LOCAL_IPS
         self.widget = self.widget_factory(config=self.config,
                                         local_kernel=local_kernel)
         self.widget._existing = self.existing
         self.widget._may_close = not self.existing
         self.widget._confirm_exit = self.confirm_exit
 
         self.widget.kernel_manager = self.kernel_manager
         self.window = MainWindow(self.app,
                                 confirm_exit=self.confirm_exit,
                                 new_frontend_factory=self.new_frontend_master,
                                 slave_frontend_factory=self.new_frontend_slave,
                                 )
         self.window.log = self.log
         self.window.add_tab_with_frontend(self.widget)
         self.window.init_menu_bar()
 
         self.window.setWindowTitle('Python' if self.pure else 'IPython')
 
     def init_colors(self):
         """Configure the coloring of the widget"""
         # Note: This will be dramatically simplified when colors
         # are removed from the backend.
 
         if self.pure:
             # only IPythonWidget supports styling
             return
 
         # parse the colors arg down to current known labels
         try:
             colors = self.config.ZMQInteractiveShell.colors
         except AttributeError:
             colors = None
         try:
             style = self.config.IPythonWidget.syntax_style
         except AttributeError:
             style = None
 
         # find the value for colors:
         if colors:
             colors=colors.lower()
             if colors in ('lightbg', 'light'):
                 colors='lightbg'
             elif colors in ('dark', 'linux'):
                 colors='linux'
             else:
                 colors='nocolor'
         elif style:
             if style=='bw':
                 colors='nocolor'
             elif styles.dark_style(style):
                 colors='linux'
             else:
                 colors='lightbg'
         else:
             colors=None
 
         # Configure the style.
         widget = self.widget
         if style:
             widget.style_sheet = styles.sheet_from_template(style, colors)
             widget.syntax_style = style
             widget._syntax_style_changed()
             widget._style_sheet_changed()
         elif colors:
             # use a default style
             widget.set_default_style(colors=colors)
         else:
             # this is redundant for now, but allows the widget's
             # defaults to change
             widget.set_default_style()
 
         if self.stylesheet:
             # we got an expicit stylesheet
             if os.path.isfile(self.stylesheet):
                 with open(self.stylesheet) as f:
                     sheet = f.read()
                 widget.style_sheet = sheet
                 widget._style_sheet_changed()
             else:
                 raise IOError("Stylesheet %r not found."%self.stylesheet)
 
+    def init_signal(self):
+        """allow clean shutdown on sigint"""
+        signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-2))
+        # need a timer, so that QApplication doesn't block until a real
+        # Qt event fires (can require mouse movement)
+        # timer trick from http://stackoverflow.com/q/4938723/938949
+        timer = QtCore.QTimer()
+         # Let the interpreter run each 200 ms:
+        timer.timeout.connect(lambda: None)
+        timer.start(200)
+        # hold onto ref, so the timer doesn't get cleaned up
+        self._sigint_timer = timer
+
     @catch_config_error
     def initialize(self, argv=None):
         super(IPythonQtConsoleApp, self).initialize(argv)
-        self.init_connection_file()
-        default_secure(self.config)
-        self.init_ssh()
-        self.init_kernel_manager()
+        IPythonConsoleApp.initialize(self,argv)
         self.init_qt_elements()
         self.init_colors()
+        self.init_signal()
 
     def start(self):
 
         # draw the window
         self.window.show()
 
         # Start the application main loop.
         self.app.exec_()
 
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 def main():
     app = IPythonQtConsoleApp()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()

IPython/frontend/terminal/ipapp.py

@@ -1,399 +1,407
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The :class:`~IPython.core.application.Application` object for the command
 line :command:`ipython` program.
 
 Authors
 -------
 
 * Brian Granger
 * Fernando Perez
 * Min Ragan-Kelley
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
 import logging
 import os
 import sys
 
 from IPython.config.loader import (
     Config, PyFileConfigLoader, ConfigFileNotFound
 )
 from IPython.config.application import boolean_flag, catch_config_error
 from IPython.core import release
 from IPython.core import usage
 from IPython.core.completer import IPCompleter
 from IPython.core.crashhandler import CrashHandler
 from IPython.core.formatters import PlainTextFormatter
 from IPython.core.prompts import PromptManager
 from IPython.core.application import (
     ProfileDir, BaseIPythonApplication, base_flags, base_aliases
 )
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
 from IPython.lib import inputhook
 from IPython.utils import warn
 from IPython.utils.path import get_ipython_dir, check_for_old_config
 from IPython.utils.traitlets import (
     Bool, List, Dict, CaselessStrEnum
 )
 
 #-----------------------------------------------------------------------------
 # Globals, utilities and helpers
 #-----------------------------------------------------------------------------
 
 #: The default config file name for this application.
 default_config_file_name = u'ipython_config.py'
 
 _examples = """
 ipython --pylab            # start in pylab mode
 ipython --pylab=qt         # start in pylab mode with the qt4 backend
 ipython --log-level=DEBUG  # set logging to DEBUG
 ipython --profile=foo      # start with profile foo
 
 ipython qtconsole          # start the qtconsole GUI application
 ipython qtconsole -h       # show the help string for the qtconsole subcmd
 
+ipython console            # start the terminal-based console application
+ipython console -h         # show the help string for the console subcmd
+
 ipython profile create foo # create profile foo w/ default config files
 ipython profile -h         # show the help string for the profile subcmd
 """
 
 #-----------------------------------------------------------------------------
 # Crash handler for this application
 #-----------------------------------------------------------------------------
 
 class IPAppCrashHandler(CrashHandler):
     """sys.excepthook for IPython itself, leaves a detailed report on disk."""
 
     def __init__(self, app):
         contact_name = release.authors['Fernando'][0]
         contact_email = release.author_email
         bug_tracker = 'https://github.com/ipython/ipython/issues'
         super(IPAppCrashHandler,self).__init__(
             app, contact_name, contact_email, bug_tracker
         )
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
         # Start with parent report
         report = [super(IPAppCrashHandler, self).make_report(traceback)]
         # Add interactive-specific info we may have
         rpt_add = report.append
         try:
             rpt_add(sec_sep+"History of session input:")
             for line in self.app.shell.user_ns['_ih']:
                 rpt_add(line)
             rpt_add('\n*** Last line of input (may not be in above history):\n')
             rpt_add(self.app.shell._last_input_line+'\n')
         except:
             pass
 
         return ''.join(report)
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 flags = dict(base_flags)
 flags.update(shell_flags)
-addflag = lambda *args: flags.update(boolean_flag(*args))
+frontend_flags = {}
+addflag = lambda *args: frontend_flags.update(boolean_flag(*args))
 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
         'Turn on auto editing of files with syntax errors.',
         'Turn off auto editing of files with syntax errors.'
 )
 addflag('banner', 'TerminalIPythonApp.display_banner',
         "Display a banner upon starting IPython.",
         "Don't display a banner upon starting IPython."
 )
 addflag('confirm-exit', 'TerminalInteractiveShell.confirm_exit',
     """Set to confirm when you try to exit IPython with an EOF (Control-D
     in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
     you can force a direct exit without any confirmation.""",
     "Don't prompt the user when exiting."
 )
 addflag('term-title', 'TerminalInteractiveShell.term_title',
     "Enable auto setting the terminal title.",
     "Disable auto setting the terminal title."
 )
 classic_config = Config()
 classic_config.InteractiveShell.cache_size = 0
 classic_config.PlainTextFormatter.pprint = False
 classic_config.PromptManager.in_template = '>>> '
 classic_config.PromptManager.in2_template = '... '
 classic_config.PromptManager.out_template = ''
 classic_config.InteractiveShell.separate_in = ''
 classic_config.InteractiveShell.separate_out = ''
 classic_config.InteractiveShell.separate_out2 = ''
 classic_config.InteractiveShell.colors = 'NoColor'
 classic_config.InteractiveShell.xmode = 'Plain'
 
-flags['classic']=(
+frontend_flags['classic']=(
     classic_config,
     "Gives IPython a similar feel to the classic Python prompt."
 )
 # # log doesn't make so much sense this way anymore
 # paa('--log','-l',
 #     action='store_true', dest='InteractiveShell.logstart',
 #     help="Start logging to the default log file (./ipython_log.py).")
 #
 # # quick is harder to implement
-flags['quick']=(
+frontend_flags['quick']=(
     {'TerminalIPythonApp' : {'quick' : True}},
     "Enable quick startup with no config files."
 )
 
-flags['i'] = (
+frontend_flags['i'] = (
     {'TerminalIPythonApp' : {'force_interact' : True}},
     """If running code from the command line, become interactive afterwards.
     Note: can also be given simply as '-i.'"""
 )
-flags['pylab'] = (
+frontend_flags['pylab'] = (
     {'TerminalIPythonApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
+flags.update(frontend_flags)
 
 aliases = dict(base_aliases)
 aliases.update(shell_aliases)
 
 # it's possible we don't want short aliases for *all* of these:
 aliases.update(dict(
     gui='TerminalIPythonApp.gui',
     pylab='TerminalIPythonApp.pylab',
 ))
 
 #-----------------------------------------------------------------------------
 # Main classes and functions
 #-----------------------------------------------------------------------------
 
 class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):
     name = u'ipython'
     description = usage.cl_usage
     default_config_file_name = default_config_file_name
     crash_handler_class = IPAppCrashHandler
     examples = _examples
 
     flags = Dict(flags)
     aliases = Dict(aliases)
     classes = List()
     def _classes_default(self):
         """This has to be in a method, for TerminalIPythonApp to be available."""
         return [
             InteractiveShellApp, # ShellApp comes before TerminalApp, because
             self.__class__,      # it will also affect subclasses (e.g. QtConsole)
             TerminalInteractiveShell,
             PromptManager,
             ProfileDir,
             PlainTextFormatter,
             IPCompleter,
         ]
 
     subcommands = Dict(dict(
         qtconsole=('IPython.frontend.qt.console.qtconsoleapp.IPythonQtConsoleApp',
             """Launch the IPython Qt Console."""
         ),
         notebook=('IPython.frontend.html.notebook.notebookapp.NotebookApp',
-            """Launch the IPython HTML Notebook Server"""
+            """Launch the IPython HTML Notebook Server."""
         ),
         profile = ("IPython.core.profileapp.ProfileApp",
             "Create and manage IPython profiles."
         ),
         kernel = ("IPython.zmq.ipkernel.IPKernelApp",
             "Start a kernel without an attached frontend."
         ),
+        console=('IPython.frontend.terminal.console.app.ZMQTerminalIPythonApp',
+            """Launch the IPython terminal-based Console."""
+        ),
     ))
 
     # *do* autocreate requested profile, but don't create the config file.
     auto_create=Bool(True)
     # configurables
     ignore_old_config=Bool(False, config=True,
         help="Suppress warning messages about legacy config files"
     )
     quick = Bool(False, config=True,
         help="""Start IPython quickly by skipping the loading of config files."""
     )
     def _quick_changed(self, name, old, new):
         if new:
             self.load_config_file = lambda *a, **kw: None
             self.ignore_old_config=True
 
     gui = CaselessStrEnum(('qt', 'wx', 'gtk', 'glut', 'pyglet'), config=True,
         help="Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet')."
     )
     pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'auto'],
         config=True,
         help="""Pre-load matplotlib and numpy for interactive use,
         selecting a particular matplotlib backend and loop integration.
         """
     )
     display_banner = Bool(True, config=True,
         help="Whether to display a banner upon starting IPython."
     )
 
     # if there is code of files to run from the cmd line, don't interact
     # unless the --i flag (App.force_interact) is true.
     force_interact = Bool(False, config=True,
         help="""If a command or file is given via the command-line,
         e.g. 'ipython foo.py"""
     )
     def _force_interact_changed(self, name, old, new):
         if new:
             self.interact = True
 
     def _file_to_run_changed(self, name, old, new):
         if new and not self.force_interact:
                 self.interact = False
     _code_to_run_changed = _file_to_run_changed
 
     # internal, not-configurable
     interact=Bool(True)
 
 
     def parse_command_line(self, argv=None):
         """override to allow old '-pylab' flag with deprecation warning"""
 
         argv = sys.argv[1:] if argv is None else argv
 
         if '-pylab' in argv:
             # deprecated `-pylab` given,
             # warn and transform into current syntax
             argv = argv[:] # copy, don't clobber
             idx = argv.index('-pylab')
             warn.warn("`-pylab` flag has been deprecated.\n"
             "    Use `--pylab` instead, or `--pylab=foo` to specify a backend.")
             sub = '--pylab'
             if len(argv) > idx+1:
                 # check for gui arg, as in '-pylab qt'
                 gui = argv[idx+1]
                 if gui in ('wx', 'qt', 'qt4', 'gtk', 'auto'):
                     sub = '--pylab='+gui
                     argv.pop(idx+1)
             argv[idx] = sub
 
         return super(TerminalIPythonApp, self).parse_command_line(argv)
     
     @catch_config_error
     def initialize(self, argv=None):
         """Do actions after construct, but before starting the app."""
         super(TerminalIPythonApp, self).initialize(argv)
         if self.subapp is not None:
             # don't bother initializing further, starting subapp
             return
         if not self.ignore_old_config:
             check_for_old_config(self.ipython_dir)
         # print self.extra_args
         if self.extra_args:
             self.file_to_run = self.extra_args[0]
         # create the shell
         self.init_shell()
         # and draw the banner
         self.init_banner()
         # Now a variety of things that happen after the banner is printed.
         self.init_gui_pylab()
         self.init_extensions()
         self.init_code()
 
     def init_shell(self):
         """initialize the InteractiveShell instance"""
         # I am a little hesitant to put these into InteractiveShell itself.
         # But that might be the place for them
         sys.path.insert(0, '')
 
         # Create an InteractiveShell instance.
         # shell.display_banner should always be False for the terminal
         # based app, because we call shell.show_banner() by hand below
         # so the banner shows *before* all extension loading stuff.
         self.shell = TerminalInteractiveShell.instance(config=self.config,
                         display_banner=False, profile_dir=self.profile_dir,
                         ipython_dir=self.ipython_dir)
         self.shell.configurables.append(self)
 
     def init_banner(self):
         """optionally display the banner"""
         if self.display_banner and self.interact:
             self.shell.show_banner()
         # Make sure there is a space below the banner.
         if self.log_level <= logging.INFO: print
 
 
     def init_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
         gui = self.gui
 
         # Using `pylab` will also require gui activation, though which toolkit
         # to use may be chosen automatically based on mpl configuration.
         if self.pylab:
             activate = self.shell.enable_pylab
             if self.pylab == 'auto':
                 gui = None
             else:
                 gui = self.pylab
         else:
             # Enable only GUI integration, no pylab
             activate = inputhook.enable_gui
 
         if gui or self.pylab:
             try:
                 self.log.info("Enabling GUI event loop integration, "
                               "toolkit=%s, pylab=%s" % (gui, self.pylab) )
                 if self.pylab:
                     activate(gui, import_all=self.pylab_import_all)
                 else:
                     activate(gui)
             except:
                 self.log.warn("Error in enabling GUI event loop integration:")
                 self.shell.showtraceback()
 
     def start(self):
         if self.subapp is not None:
             return self.subapp.start()
         # perform any prexec steps:
         if self.interact:
             self.log.debug("Starting IPython's mainloop...")
             self.shell.mainloop()
         else:
             self.log.debug("IPython not interactive...")
 
 
 def load_default_config(ipython_dir=None):
     """Load the default config file from the default ipython_dir.
 
     This is useful for embedded shells.
     """
     if ipython_dir is None:
         ipython_dir = get_ipython_dir()
     profile_dir = os.path.join(ipython_dir, 'profile_default')
     cl = PyFileConfigLoader(default_config_file_name, profile_dir)
     try:
         config = cl.load_config()
     except ConfigFileNotFound:
         # no config found
         config = Config()
     return config
 
 
 def launch_new_instance():
     """Create and run a full blown IPython instance"""
     app = TerminalIPythonApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     launch_new_instance()

IPython/lib/kernel.py

@@ -1,255 +1,315
 """Utilities for connecting to kernels
 
 Authors:
 
 * Min Ragan-Kelley
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import glob
 import json
 import os
 import sys
 from getpass import getpass
 from subprocess import Popen, PIPE
 
 # external imports
 from IPython.external.ssh import tunnel
 
 # IPython imports
 from IPython.core.profiledir import ProfileDir
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils.py3compat import str_to_bytes
 
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def get_connection_file(app=None):
     """Return the path to the connection file of an app
     
     Parameters
     ----------
     app : KernelApp instance [optional]
         If unspecified, the currently running app will be used
     """
     if app is None:
         from IPython.zmq.kernelapp import KernelApp
         if not KernelApp.initialized():
             raise RuntimeError("app not specified, and not in a running Kernel")
 
         app = KernelApp.instance()
     return filefind(app.connection_file, ['.', app.profile_dir.security_dir])
 
 def find_connection_file(filename, profile=None):
     """find a connection file, and return its absolute path.
     
     The current working directory and the profile's security
     directory will be searched for the file if it is not given by
     absolute path.
     
     If profile is unspecified, then the current running application's
     profile will be used, or 'default', if not run from IPython.
     
     If the argument does not match an existing file, it will be interpreted as a
     fileglob, and the matching file in the profile's security dir with
     the latest access time will be used.
     
     Parameters
     ----------
     filename : str
         The connection file or fileglob to search for.
     profile : str [optional]
         The name of the profile to use when searching for the connection file,
         if different from the current IPython session or 'default'.
     
     Returns
     -------
     str : The absolute path of the connection file.
     """
     from IPython.core.application import BaseIPythonApplication as IPApp
     try:
         # quick check for absolute path, before going through logic
         return filefind(filename)
     except IOError:
         pass
     
     if profile is None:
         # profile unspecified, check if running from an IPython app
         if IPApp.initialized():
             app = IPApp.instance()
             profile_dir = app.profile_dir
         else:
             # not running in IPython, use default profile
             profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), 'default')
     else:
         # find profiledir by profile name:
         profile_dir = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
     security_dir = profile_dir.security_dir
     
     try:
         # first, try explicit name
         return filefind(filename, ['.', security_dir])
     except IOError:
         pass
     
     # not found by full name
     
     if '*' in filename:
         # given as a glob already
         pat = filename
     else:
         # accept any substring match
         pat = '*%s*' % filename
     matches = glob.glob( os.path.join(security_dir, pat) )
     if not matches:
         raise IOError("Could not find %r in %r" % (filename, security_dir))
     elif len(matches) == 1:
         return matches[0]
     else:
         # get most recent match, by access time:
         return sorted(matches, key=lambda f: os.stat(f).st_atime)[-1]
 
 def get_connection_info(connection_file=None, unpack=False, profile=None):
     """Return the connection information for the current Kernel.
     
     Parameters
     ----------
     connection_file : str [optional]
         The connection file to be used. Can be given by absolute path, or
         IPython will search in the security directory of a given profile.
         If run from IPython, 
         
         If unspecified, the connection file for the currently running
         IPython Kernel will be used, which is only allowed from inside a kernel.
     unpack : bool [default: False]
         if True, return the unpacked dict, otherwise just the string contents
         of the file.
     profile : str [optional]
         The name of the profile to use when searching for the connection file,
         if different from the current IPython session or 'default'.
         
     
     Returns
     -------
     The connection dictionary of the current kernel, as string or dict,
     depending on `unpack`.
     """
     if connection_file is None:
         # get connection file from current kernel
         cf = get_connection_file()
     else:
         # connection file specified, allow shortnames:
         cf = find_connection_file(connection_file, profile=profile)
     
     with open(cf) as f:
         info = f.read()
     
     if unpack:
         info = json.loads(info)
         # ensure key is bytes:
         info['key'] = str_to_bytes(info.get('key', ''))
     return info
 
 def connect_qtconsole(connection_file=None, argv=None, profile=None):
     """Connect a qtconsole to the current kernel.
     
     This is useful for connecting a second qtconsole to a kernel, or to a
     local notebook.
     
     Parameters
     ----------
     connection_file : str [optional]
         The connection file to be used. Can be given by absolute path, or
         IPython will search in the security directory of a given profile.
         If run from IPython, 
         
         If unspecified, the connection file for the currently running
         IPython Kernel will be used, which is only allowed from inside a kernel.
     argv : list [optional]
         Any extra args to be passed to the console.
     profile : str [optional]
         The name of the profile to use when searching for the connection file,
         if different from the current IPython session or 'default'.
     
     
     Returns
     -------
     subprocess.Popen instance running the qtconsole frontend
     """
     argv = [] if argv is None else argv
     
     if connection_file is None:
         # get connection file from current kernel
         cf = get_connection_file()
     else:
         cf = find_connection_file(connection_file, profile=profile)
     
     cmd = ';'.join([
         "from IPython.frontend.qt.console import qtconsoleapp",
         "qtconsoleapp.main()"
     ])
     
     return Popen([sys.executable, '-c', cmd, '--existing', cf] + argv, stdout=PIPE, stderr=PIPE)
 
 def tunnel_to_kernel(connection_info, sshserver, sshkey=None):
     """tunnel connections to a kernel via ssh
     
     This will open four SSH tunnels from localhost on this machine to the
     ports associated with the kernel.  They can be either direct
     localhost-localhost tunnels, or if an intermediate server is necessary,
     the kernel must be listening on a public IP.
     
     Parameters
     ----------
     connection_info : dict or str (path)
         Either a connection dict, or the path to a JSON connection file
     sshserver : str
         The ssh sever to use to tunnel to the kernel. Can be a full
         `user@server:port` string. ssh config aliases are respected.
     sshkey : str [optional]
         Path to file containing ssh key to use for authentication.
         Only necessary if your ssh config does not already associate
         a keyfile with the host.
     
     Returns
     -------
     
     (shell, iopub, stdin, hb) : ints
         The four ports on localhost that have been forwarded to the kernel.
     """
     if isinstance(connection_info, basestring):
         # it's a path, unpack it
         with open(connection_info) as f:
             connection_info = json.loads(f.read())
     
     cf = connection_info
     
     lports = tunnel.select_random_ports(4)
     rports = cf['shell_port'], cf['iopub_port'], cf['stdin_port'], cf['hb_port']
     
     remote_ip = cf['ip']
     
     if tunnel.try_passwordless_ssh(sshserver, sshkey):
         password=False
     else:
         password = getpass("SSH Password for %s: "%sshserver)
     
     for lp,rp in zip(lports, rports):
         tunnel.ssh_tunnel(lp, rp, sshserver, remote_ip, sshkey, password)
     
     return tuple(lports)
     
 
+def swallow_argv(argv, aliases=None, flags=None):
+    """strip frontend-specific aliases and flags from an argument list
+    
+    For use primarily in frontend apps that want to pass a subset of command-line
+    arguments through to a subprocess, where frontend-specific flags and aliases
+    should be removed from the list.
+    
+    Parameters
+    ----------
+    
+    argv : list(str)
+        The starting argv, to be filtered
+    aliases : container of aliases (dict, list, set, etc.)
+        The frontend-specific aliases to be removed
+    flags : container of flags (dict, list, set, etc.)
+        The frontend-specific flags to be removed
+    
+    Returns
+    -------
+    
+    argv : list(str)
+        The argv list, excluding flags and aliases that have been stripped
+    """
+    
+    if aliases is None:
+        aliases = set()
+    if flags is None:
+        flags = set()
+    
+    stripped = list(argv) # copy
+    
+    swallow_next = False
+    was_flag = False
+    for a in argv:
+        if swallow_next:
+            swallow_next = False
+            # last arg was an alias, remove the next one
+            # *unless* the last alias has a no-arg flag version, in which
+            # case, don't swallow the next arg if it's also a flag:
+            if not (was_flag and a.startswith('-')):
+                stripped.remove(a)
+                continue
+        if a.startswith('-'):
+            split = a.lstrip('-').split('=')
+            alias = split[0]
+            if alias in aliases:
+                stripped.remove(a)
+                if len(split) == 1:
+                    # alias passed with arg via space
+                    swallow_next = True
+                    # could have been a flag that matches an alias, e.g. `existing`
+                    # in which case, we might not swallow the next arg
+                    was_flag = alias in flags
+            elif alias in flags and len(split) == 1:
+                # strip flag, but don't swallow next, as flags don't take args
+                stripped.remove(a)
+    
+    # return shortened list
+    return stripped
+

IPython/testing/_paramtestpy2.py

@@ -1,96 +1,97
 """Implementation of the parametric test support for Python 2.x
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2009-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
+import sys
 import unittest
 from compiler.consts import CO_GENERATOR
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def isgenerator(func):
     try:
         return func.func_code.co_flags & CO_GENERATOR != 0
     except AttributeError:
         return False
 
 class ParametricTestCase(unittest.TestCase):
     """Write parametric tests in normal unittest testcase form.
 
     Limitations: the last iteration misses printing out a newline when running
     in verbose mode.
     """
     def run_parametric(self, result, testMethod):
         # But if we have a test generator, we iterate it ourselves
         testgen = testMethod()
         while True:
             try:
                 # Initialize test
                 result.startTest(self)
 
                 # SetUp
                 try:
                     self.setUp()
                 except KeyboardInterrupt:
                     raise
                 except:
-                    result.addError(self, self._exc_info())
+                    result.addError(self, sys.exc_info())
                     return
                 # Test execution
                 ok = False
                 try:
                     testgen.next()
                     ok = True
                 except StopIteration:
                     # We stop the loop
                     break
                 except self.failureException:
-                    result.addFailure(self, self._exc_info())
+                    result.addFailure(self, sys.exc_info())
                 except KeyboardInterrupt:
                     raise
                 except:
-                    result.addError(self, self._exc_info())
+                    result.addError(self, sys.exc_info())
                 # TearDown
                 try:
                     self.tearDown()
                 except KeyboardInterrupt:
                     raise
                 except:
-                    result.addError(self, self._exc_info())
+                    result.addError(self, sys.exc_info())
                     ok = False
                 if ok: result.addSuccess(self)
 
             finally:
                 result.stopTest(self)
 
     def run(self, result=None):
         if result is None:
             result = self.defaultTestResult()
         testMethod = getattr(self, self._testMethodName)
         # For normal tests, we just call the base class and return that
         if isgenerator(testMethod):
             return self.run_parametric(result, testMethod)
         else:
             return super(ParametricTestCase, self).run(result)
 
 
 def parametric(func):
     """Decorator to make a simple function into a normal test via unittest."""
 
     class Tester(ParametricTestCase):
         test = staticmethod(func)
 
     Tester.__name__ = func.__name__
 
     return Tester

IPython/zmq/blockingkernelmanager.py

@@ -1,129 +1,154
 """Implement a fully blocking kernel manager.
 
 Useful for test suites and blocking terminal interfaces.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING.txt, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 from Queue import Queue, Empty
+from threading import Event
 
 # Our own
 from IPython.utils import io
 from IPython.utils.traitlets import Type
 
 from .kernelmanager import (KernelManager, SubSocketChannel, HBSocketChannel,
                            ShellSocketChannel, StdInSocketChannel)
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 class BlockingSubSocketChannel(SubSocketChannel):
 
     def __init__(self, context, session, address=None):
         super(BlockingSubSocketChannel, self).__init__(context, session,
                                                        address)
         self._in_queue = Queue()
 
     def call_handlers(self, msg):
         #io.rprint('[[Sub]]', msg) # dbg
         self._in_queue.put(msg)
 
     def msg_ready(self):
         """Is there a message that has been received?"""
         if self._in_queue.qsize() == 0:
             return False
         else:
             return True
 
     def get_msg(self, block=True, timeout=None):
         """Get a message if there is one that is ready."""
         if block and timeout is None:
             # never use timeout=None, because get
             # becomes uninterruptible
             timeout = 1e6
         return self._in_queue.get(block, timeout)
 
     def get_msgs(self):
         """Get all messages that are currently ready."""
         msgs = []
         while True:
             try:
                 msgs.append(self.get_msg(block=False))
             except Empty:
                 break
         return msgs
 
 
 class BlockingShellSocketChannel(ShellSocketChannel):
 
     def __init__(self, context, session, address=None):
         super(BlockingShellSocketChannel, self).__init__(context, session,
                                                         address)
         self._in_queue = Queue()
 
     def call_handlers(self, msg):
         #io.rprint('[[Shell]]', msg) # dbg
         self._in_queue.put(msg)
 
     def msg_ready(self):
         """Is there a message that has been received?"""
         if self._in_queue.qsize() == 0:
             return False
         else:
             return True
 
     def get_msg(self, block=True, timeout=None):
         """Get a message if there is one that is ready."""
         if block and timeout is None:
             # never use timeout=None, because get
             # becomes uninterruptible
             timeout = 1e6
         return self._in_queue.get(block, timeout)
 
     def get_msgs(self):
         """Get all messages that are currently ready."""
         msgs = []
         while True:
             try:
                 msgs.append(self.get_msg(block=False))
             except Empty:
                 break
         return msgs
     
 
 class BlockingStdInSocketChannel(StdInSocketChannel):
     
+    def __init__(self, context, session, address=None):
+        super(BlockingStdInSocketChannel, self).__init__(context, session, address)
+        self._in_queue = Queue()
+        
     def call_handlers(self, msg):
         #io.rprint('[[Rep]]', msg) # dbg
-        pass
+        self._in_queue.put(msg)
+        
+    def get_msg(self, block=True, timeout=None):
+        "Gets a message if there is one that is ready."
+        return self._in_queue.get(block, timeout)
+        
+    def get_msgs(self):
+        """Get all messages that are currently ready."""
+        msgs = []
+        while True:
+            try:
+                msgs.append(self.get_msg(block=False))
+            except Empty:
+                break
+        return msgs
+    
+    def msg_ready(self):
+        "Is there a message that has been received?"
+        return not self._in_queue.empty()
 
 
 class BlockingHBSocketChannel(HBSocketChannel):
     
-    # This kernel needs rapid monitoring capabilities
-    time_to_dead = 0.2
+    # This kernel needs quicker monitoring, shorten to 1 sec.
+    # less than 0.5s is unreliable, and will get occasional
+    # false reports of missed beats.
+    time_to_dead = 1.
 
     def call_handlers(self, since_last_heartbeat):
-        #io.rprint('[[Heart]]', since_last_heartbeat) # dbg
+        """pause beating on missed heartbeat"""
         pass
 
 
 class BlockingKernelManager(KernelManager):
     
     # The classes to use for the various channels.
     shell_channel_class = Type(BlockingShellSocketChannel)
     sub_channel_class = Type(BlockingSubSocketChannel)
     stdin_channel_class = Type(BlockingStdInSocketChannel)
     hb_channel_class = Type(BlockingHBSocketChannel)
   

IPython/zmq/ipkernel.py

@@ -1,635 +1,651
 #!/usr/bin/env python
 """A simple interactive kernel that talks to a frontend over 0MQ.
 
 Things to do:
 
 * Implement `set_parent` logic. Right before doing exec, the Kernel should
   call set_parent on all the PUB objects with the message about to be executed.
 * Implement random port and security key logic.
 * Implement control messages.
 * Implement event loop and poll version.
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Standard library imports.
 import __builtin__
 import atexit
 import sys
 import time
 import traceback
 import logging
-
+from signal import (
+        signal, default_int_handler, SIGINT, SIG_IGN
+)
 # System library imports.
 import zmq
 
 # Local imports.
 from IPython.core import pylabtools
 from IPython.config.configurable import Configurable
 from IPython.config.application import boolean_flag, catch_config_error
 from IPython.core.application import ProfileDir
 from IPython.core.error import StdinNotImplementedError
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils.jsonutil import json_clean
 from IPython.utils.traitlets import (
     Any, Instance, Float, Dict, CaselessStrEnum
 )
 
 from entry_point import base_launch_kernel
 from kernelapp import KernelApp, kernel_flags, kernel_aliases
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(Configurable):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
 
     # attribute to override with a GUI
     eventloop = Any(None)
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     session = Instance(Session)
     profile_dir = Instance('IPython.core.profiledir.ProfileDir')
     shell_socket = Instance('zmq.Socket')
     iopub_socket = Instance('zmq.Socket')
     stdin_socket = Instance('zmq.Socket')
     log = Instance(logging.Logger)
 
     # Private interface
 
     # Time to sleep after flushing the stdout/err buffers in each execute
     # cycle.  While this introduces a hard limit on the minimal latency of the
     # execute cycle, it helps prevent output synchronization problems for
     # clients.
     # Units are in seconds.  The minimum zmq latency on local host is probably
     # ~150 microseconds, set this to 500us for now.  We may need to increase it
     # a little if it's not enough after more interactive testing.
     _execute_sleep = Float(0.0005, config=True)
 
     # Frequency of the kernel's event loop.
     # Units are in seconds, kernel subclasses for GUI toolkits may need to
     # adapt to milliseconds.
     _poll_interval = Float(0.05, config=True)
 
     # If the shutdown was requested over the network, we leave here the
     # necessary reply message so it can be sent by our registered atexit
     # handler.  This ensures that the reply is only sent to clients truly at
     # the end of our shutdown process (which happens after the underlying
     # IPython shell's own shutdown).
     _shutdown_message = None
 
     # This is a dict of port number that the kernel is listening on. It is set
     # by record_ports and used by connect_request.
     _recorded_ports = Dict()
 
 
 
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
 
         # Before we even start up the shell, register *first* our exit handlers
         # so they come before the shell's
         atexit.register(self._at_shutdown)
 
         # Initialize the InteractiveShell subclass
         self.shell = ZMQInteractiveShell.instance(config=self.config,
             profile_dir = self.profile_dir,
         )
         self.shell.displayhook.session = self.session
         self.shell.displayhook.pub_socket = self.iopub_socket
         self.shell.display_pub.session = self.session
         self.shell.display_pub.pub_socket = self.iopub_socket
 
         # TMP - hack while developing
         self.shell._reply_content = None
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request',
                       'object_info_request', 'history_request',
                       'connect_request', 'shutdown_request']
         self.handlers = {}
         for msg_type in msg_types:
             self.handlers[msg_type] = getattr(self, msg_type)
 
     def do_one_iteration(self):
         """Do one iteration of the kernel's evaluation loop.
         """
         try:
             ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
         except Exception:
             self.log.warn("Invalid Message:", exc_info=True)
             return
         if msg is None:
             return
 
         msg_type = msg['header']['msg_type']
 
         # This assert will raise in versions of zeromq 2.0.7 and lesser.
         # We now require 2.0.8 or above, so we can uncomment for safety.
         # print(ident,msg, file=sys.__stdout__)
         assert ident is not None, "Missing message part."
 
         # Print some info about this message and leave a '--->' marker, so it's
         # easier to trace visually the message chain when debugging.  Each
         # handler prints its message at the end.
         self.log.debug('\n*** MESSAGE TYPE:'+str(msg_type)+'***')
         self.log.debug('   Content: '+str(msg['content'])+'\n   --->\n   ')
 
         # Find and call actual handler for message
         handler = self.handlers.get(msg_type, None)
         if handler is None:
             self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg))
         else:
             handler(ident, msg)
 
         # Check whether we should exit, in case the incoming message set the
         # exit flag on
         if self.shell.exit_now:
             self.log.debug('\nExiting IPython kernel...')
             # We do a normal, clean exit, which allows any actions registered
             # via atexit (such as history saving) to take place.
             sys.exit(0)
 
 
     def start(self):
         """ Start the kernel main loop.
         """
+        # a KeyboardInterrupt (SIGINT) can occur on any python statement, so
+        # let's ignore (SIG_IGN) them until we're in a place to handle them properly
+        signal(SIGINT,SIG_IGN)
         poller = zmq.Poller()
         poller.register(self.shell_socket, zmq.POLLIN)
         # loop while self.eventloop has not been overridden
         while self.eventloop is None:
             try:
                 # scale by extra factor of 10, because there is no
                 # reason for this to be anything less than ~ 0.1s
                 # since it is a real poller and will respond
                 # to events immediately
 
                 # double nested try/except, to properly catch KeyboardInterrupt
                 # due to pyzmq Issue #130
                 try:
                     poller.poll(10*1000*self._poll_interval)
+                    # restore raising of KeyboardInterrupt
+                    signal(SIGINT, default_int_handler)
                     self.do_one_iteration()
                 except:
                     raise
+                finally:
+                    # prevent raising of KeyboardInterrupt
+                    signal(SIGINT,SIG_IGN)
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 io.raw_print("KeyboardInterrupt caught in kernel")
+        # stop ignoring sigint, now that we are out of our own loop,
+        # we don't want to prevent future code from handling it
+        signal(SIGINT, default_int_handler)
         if self.eventloop is not None:
             try:
                 self.eventloop(self)
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 io.raw_print("KeyboardInterrupt caught in kernel")
 
 
     def record_ports(self, ports):
         """Record the ports that this kernel is using.
 
         The creator of the Kernel instance must call this methods if they
         want the :meth:`connect_request` method to return the port numbers.
         """
         self._recorded_ports = ports
 
     #---------------------------------------------------------------------------
     # Kernel request handlers
     #---------------------------------------------------------------------------
 
     def _publish_pyin(self, code, parent):
         """Publish the code request on the pyin stream."""
 
         self.session.send(self.iopub_socket, u'pyin', {u'code':code},
                           parent=parent)
 
     def execute_request(self, ident, parent):
 
         self.session.send(self.iopub_socket,
                           u'status',
                           {u'execution_state':u'busy'},
                           parent=parent )
         
         try:
             content = parent[u'content']
             code = content[u'code']
             silent = content[u'silent']
         except:
             self.log.error("Got bad msg: ")
             self.log.error(str(Message(parent)))
             return
 
         shell = self.shell # we'll need this a lot here
 
         # Replace raw_input. Note that is not sufficient to replace
         # raw_input in the user namespace.
         if content.get('allow_stdin', False):
             raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
         else:
             raw_input = lambda prompt='' : self._no_raw_input()
 
         if py3compat.PY3:
             __builtin__.input = raw_input
         else:
             __builtin__.raw_input = raw_input
 
         # Set the parent message of the display hook and out streams.
         shell.displayhook.set_parent(parent)
         shell.display_pub.set_parent(parent)
         sys.stdout.set_parent(parent)
         sys.stderr.set_parent(parent)
 
         # Re-broadcast our input for the benefit of listening clients, and
         # start computing output
         if not silent:
             self._publish_pyin(code, parent)
 
         reply_content = {}
         try:
             if silent:
                 # run_code uses 'exec' mode, so no displayhook will fire, and it
                 # doesn't call logging or history manipulations.  Print
                 # statements in that code will obviously still execute.
                 shell.run_code(code)
             else:
                 # FIXME: the shell calls the exception handler itself.
                 shell.run_cell(code, store_history=True)
         except:
             status = u'error'
             # FIXME: this code right now isn't being used yet by default,
             # because the run_cell() call above directly fires off exception
             # reporting.  This code, therefore, is only active in the scenario
             # where runlines itself has an unhandled exception.  We need to
             # uniformize this, for all exception construction to come from a
             # single location in the codbase.
             etype, evalue, tb = sys.exc_info()
             tb_list = traceback.format_exception(etype, evalue, tb)
             reply_content.update(shell._showtraceback(etype, evalue, tb_list))
         else:
             status = u'ok'
 
         reply_content[u'status'] = status
 
         # Return the execution counter so clients can display prompts
         reply_content['execution_count'] = shell.execution_count -1
 
         # FIXME - fish exception info out of shell, possibly left there by
         # runlines.  We'll need to clean up this logic later.
         if shell._reply_content is not None:
             reply_content.update(shell._reply_content)
             # reset after use
             shell._reply_content = None
 
         # At this point, we can tell whether the main code execution succeeded
         # or not.  If it did, we proceed to evaluate user_variables/expressions
         if reply_content['status'] == 'ok':
             reply_content[u'user_variables'] = \
                          shell.user_variables(content[u'user_variables'])
             reply_content[u'user_expressions'] = \
                          shell.user_expressions(content[u'user_expressions'])
         else:
             # If there was an error, don't even try to compute variables or
             # expressions
             reply_content[u'user_variables'] = {}
             reply_content[u'user_expressions'] = {}
 
         # Payloads should be retrieved regardless of outcome, so we can both
         # recover partial output (that could have been generated early in a
         # block, before an error) and clear the payload system always.
         reply_content[u'payload'] = shell.payload_manager.read_payload()
         # Be agressive about clearing the payload because we don't want
         # it to sit in memory until the next execute_request comes in.
         shell.payload_manager.clear_payload()
 
         # Flush output before sending the reply.
         sys.stdout.flush()
         sys.stderr.flush()
         # FIXME: on rare occasions, the flush doesn't seem to make it to the
         # clients... This seems to mitigate the problem, but we definitely need
         # to better understand what's going on.
         if self._execute_sleep:
             time.sleep(self._execute_sleep)
 
         # Send the reply.
         reply_content = json_clean(reply_content)
         reply_msg = self.session.send(self.shell_socket, u'execute_reply',
                                       reply_content, parent, ident=ident)
         self.log.debug(str(reply_msg))
 
         if reply_msg['content']['status'] == u'error':
             self._abort_queue()
 
         self.session.send(self.iopub_socket,
                           u'status',
                           {u'execution_state':u'idle'},
                           parent=parent )
 
     def complete_request(self, ident, parent):
         txt, matches = self._complete(parent)
         matches = {'matches' : matches,
                    'matched_text' : txt,
                    'status' : 'ok'}
         matches = json_clean(matches)
         completion_msg = self.session.send(self.shell_socket, 'complete_reply',
                                            matches, parent, ident)
         self.log.debug(str(completion_msg))
 
     def object_info_request(self, ident, parent):
         object_info = self.shell.object_inspect(parent['content']['oname'])
         # Before we send this object over, we scrub it for JSON usage
         oinfo = json_clean(object_info)
         msg = self.session.send(self.shell_socket, 'object_info_reply',
                                 oinfo, parent, ident)
         self.log.debug(msg)
 
     def history_request(self, ident, parent):
         # We need to pull these out, as passing **kwargs doesn't work with
         # unicode keys before Python 2.6.5.
         hist_access_type = parent['content']['hist_access_type']
         raw = parent['content']['raw']
         output = parent['content']['output']
         if hist_access_type == 'tail':
             n = parent['content']['n']
             hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
                                                             include_latest=True)
 
         elif hist_access_type == 'range':
             session = parent['content']['session']
             start = parent['content']['start']
             stop = parent['content']['stop']
             hist = self.shell.history_manager.get_range(session, start, stop,
                                                         raw=raw, output=output)
 
         elif hist_access_type == 'search':
             pattern = parent['content']['pattern']
             hist = self.shell.history_manager.search(pattern, raw=raw,
                                                      output=output) 
 
         else:
             hist = []
         content = {'history' : list(hist)}
         content = json_clean(content)
         msg = self.session.send(self.shell_socket, 'history_reply',
                                 content, parent, ident)
         self.log.debug(str(msg))
 
     def connect_request(self, ident, parent):
         if self._recorded_ports is not None:
             content = self._recorded_ports.copy()
         else:
             content = {}
         msg = self.session.send(self.shell_socket, 'connect_reply',
                                 content, parent, ident)
         self.log.debug(msg)
 
     def shutdown_request(self, ident, parent):
         self.shell.exit_now = True
         self._shutdown_message = self.session.msg(u'shutdown_reply',
                                                   parent['content'], parent)
         sys.exit(0)
 
     #---------------------------------------------------------------------------
     # Protected interface
     #---------------------------------------------------------------------------
 
     def _abort_queue(self):
         while True:
             try:
                 ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
             except Exception:
                 self.log.warn("Invalid Message:", exc_info=True)
                 continue
             if msg is None:
                 break
             else:
                 assert ident is not None, \
                        "Unexpected missing message part."
 
             self.log.debug("Aborting:\n"+str(Message(msg)))
             msg_type = msg['header']['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             reply_msg = self.session.send(self.shell_socket, reply_type,
                     {'status' : 'aborted'}, msg, ident=ident)
             self.log.debug(reply_msg)
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.1)
 
     def _no_raw_input(self):
         """Raise StdinNotImplentedError if active frontend doesn't support
         stdin."""
         raise StdinNotImplementedError("raw_input was called, but this "
                                        "frontend does not support stdin.") 
         
     def _raw_input(self, prompt, ident, parent):
         # Flush output before making the request.
         sys.stderr.flush()
         sys.stdout.flush()
 
         # Send the input request.
         content = json_clean(dict(prompt=prompt))
         self.session.send(self.stdin_socket, u'input_request', content, parent,
                           ident=ident)
 
         # Await a response.
         while True:
             try:
                 ident, reply = self.session.recv(self.stdin_socket, 0)
             except Exception:
                 self.log.warn("Invalid Message:", exc_info=True)
             else:
                 break
         try:
             value = reply['content']['value']
         except:
             self.log.error("Got bad raw_input reply: ")
             self.log.error(str(Message(parent)))
             value = ''
+        if value == '\x04':
+            # EOF
+            raise EOFError
         return value
 
     def _complete(self, msg):
         c = msg['content']
         try:
             cpos = int(c['cursor_pos'])
         except:
             # If we don't get something that we can convert to an integer, at
             # least attempt the completion guessing the cursor is at the end of
             # the text, if there's any, and otherwise of the line
             cpos = len(c['text'])
             if cpos==0:
                 cpos = len(c['line'])
         return self.shell.complete(c['text'], c['line'], cpos)
 
     def _object_info(self, context):
         symbol, leftover = self._symbol_from_context(context)
         if symbol is not None and not leftover:
             doc = getattr(symbol, '__doc__', '')
         else:
             doc = ''
         object_info = dict(docstring = doc)
         return object_info
 
     def _symbol_from_context(self, context):
         if not context:
             return None, context
 
         base_symbol_string = context[0]
         symbol = self.shell.user_ns.get(base_symbol_string, None)
         if symbol is None:
             symbol = __builtin__.__dict__.get(base_symbol_string, None)
         if symbol is None:
             return None, context
 
         context = context[1:]
         for i, name in enumerate(context):
             new_symbol = getattr(symbol, name, None)
             if new_symbol is None:
                 return symbol, context[i:]
             else:
                 symbol = new_symbol
 
         return symbol, []
 
     def _at_shutdown(self):
         """Actions taken at shutdown by the kernel, called by python's atexit.
         """
         # io.rprint("Kernel at_shutdown") # dbg
         if self._shutdown_message is not None:
             self.session.send(self.shell_socket, self._shutdown_message)
             self.session.send(self.iopub_socket, self._shutdown_message)
             self.log.debug(str(self._shutdown_message))
             # A very short sleep to give zmq time to flush its message buffers
             # before Python truly shuts down.
             time.sleep(0.01)
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags for the IPKernelApp
 #-----------------------------------------------------------------------------
 
 flags = dict(kernel_flags)
 flags.update(shell_flags)
 
 addflag = lambda *args: flags.update(boolean_flag(*args))
 
 flags['pylab'] = (
     {'IPKernelApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
 
 aliases = dict(kernel_aliases)
 aliases.update(shell_aliases)
 
 # it's possible we don't want short aliases for *all* of these:
 aliases.update(dict(
     pylab='IPKernelApp.pylab',
 ))
 
 #-----------------------------------------------------------------------------
 # The IPKernelApp class
 #-----------------------------------------------------------------------------
 
 class IPKernelApp(KernelApp, InteractiveShellApp):
     name = 'ipkernel'
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
     # configurables
     pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
         config=True,
         help="""Pre-load matplotlib and numpy for interactive use,
         selecting a particular matplotlib backend and loop integration.
         """
     )
     
     @catch_config_error
     def initialize(self, argv=None):
         super(IPKernelApp, self).initialize(argv)
         self.init_shell()
         self.init_extensions()
         self.init_code()
 
     def init_kernel(self):
 
         kernel = Kernel(config=self.config, session=self.session,
                                 shell_socket=self.shell_socket,
                                 iopub_socket=self.iopub_socket,
                                 stdin_socket=self.stdin_socket,
                                 log=self.log,
                                 profile_dir=self.profile_dir,
         )
         self.kernel = kernel
         kernel.record_ports(self.ports)
         shell = kernel.shell
         if self.pylab:
             try:
                 gui, backend = pylabtools.find_gui_and_backend(self.pylab)
                 shell.enable_pylab(gui, import_all=self.pylab_import_all)
             except Exception:
                 self.log.error("Pylab initialization failed", exc_info=True)
                 # print exception straight to stdout, because normally 
                 # _showtraceback associates the reply with an execution, 
                 # which means frontends will never draw it, as this exception 
                 # is not associated with any execute request.
                 
                 # replace pyerr-sending traceback with stdout
                 _showtraceback = shell._showtraceback
                 def print_tb(etype, evalue, stb):
                     print ("Error initializing pylab, pylab mode will not "
                            "be active", file=io.stderr)
                     print (shell.InteractiveTB.stb2text(stb), file=io.stdout)
                 shell._showtraceback = print_tb
                 
                 # send the traceback over stdout
                 shell.showtraceback(tb_offset=0)
                 
                 # restore proper _showtraceback method
                 shell._showtraceback = _showtraceback
                 
 
     def init_shell(self):
         self.shell = self.kernel.shell
         self.shell.configurables.append(self)
 
 
 #-----------------------------------------------------------------------------
 # Kernel main and launch functions
 #-----------------------------------------------------------------------------
 
 def launch_kernel(*args, **kwargs):
     """Launches a localhost IPython kernel, binding to the specified ports.
 
     This function simply calls entry_point.base_launch_kernel with the right
     first command to start an ipkernel.  See base_launch_kernel for arguments.
 
     Returns
     -------
     A tuple of form:
         (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
     return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
                               *args, **kwargs)
 
 
 def main():
     """Run an IPKernel as an application"""
     app = IPKernelApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()

IPython/zmq/kernelmanager.py

@@ -1,947 +1,962
 """Base classes to manage the interaction with a running kernel.
 
 TODO
 * Create logger to handle debugging and console messages.
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Standard library imports.
 import errno
 import json
 from subprocess import Popen
 import os
 import signal
 import sys
 from threading import Thread
 import time
 
 # System library imports.
 import zmq
 from zmq.eventloop import ioloop, zmqstream
 
 # Local imports.
 from IPython.config.loader import Config
 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
 from IPython.utils.traitlets import (
     HasTraits, Any, Instance, Type, Unicode, Integer, Bool
 )
 from IPython.utils.py3compat import str_to_bytes
 from IPython.zmq.entry_point import write_connection_file
 from session import Session
 
 #-----------------------------------------------------------------------------
 # Constants and exceptions
 #-----------------------------------------------------------------------------
 
 class InvalidPortNumber(Exception):
     pass
 
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
 
 # some utilities to validate message structure, these might get moved elsewhere
 # if they prove to have more generic utility
 
 def validate_string_list(lst):
     """Validate that the input is a list of strings.
 
     Raises ValueError if not."""
     if not isinstance(lst, list):
         raise ValueError('input %r must be a list' % lst)
     for x in lst:
         if not isinstance(x, basestring):
             raise ValueError('element %r in list must be a string' % x)
 
 
 def validate_string_dict(dct):
     """Validate that the input is a dict with string keys and values.
 
     Raises ValueError if not."""
     for k,v in dct.iteritems():
         if not isinstance(k, basestring):
             raise ValueError('key %r in dict must be a string' % k)
         if not isinstance(v, basestring):
             raise ValueError('value %r in dict must be a string' % v)
 
 
 #-----------------------------------------------------------------------------
 # ZMQ Socket Channel classes
 #-----------------------------------------------------------------------------
 
 class ZMQSocketChannel(Thread):
     """The base class for the channels that use ZMQ sockets.
     """
     context = None
     session = None
     socket = None
     ioloop = None
     stream = None
     _address = None
 
     def __init__(self, context, session, address):
         """Create a channel
 
         Parameters
         ----------
         context : :class:`zmq.Context`
             The ZMQ context to use.
         session : :class:`session.Session`
             The session to use.
         address : tuple
             Standard (ip, port) tuple that the kernel is listening on.
         """
         super(ZMQSocketChannel, self).__init__()
         self.daemon = True
 
         self.context = context
         self.session = session
         if address[1] == 0:
             message = 'The port number for a channel cannot be 0.'
             raise InvalidPortNumber(message)
         self._address = address
 
     def _run_loop(self):
         """Run my loop, ignoring EINTR events in the poller"""
         while True:
             try:
                 self.ioloop.start()
             except zmq.ZMQError as e:
                 if e.errno == errno.EINTR:
                     continue
                 else:
                     raise
             else:
                 break
 
     def stop(self):
         """Stop the channel's activity.
 
         This calls :method:`Thread.join` and returns when the thread
         terminates. :class:`RuntimeError` will be raised if
         :method:`self.start` is called again.
         """
         self.join()
 
     @property
     def address(self):
         """Get the channel's address as an (ip, port) tuple.
 
         By the default, the address is (localhost, 0), where 0 means a random
         port.
         """
         return self._address
 
     def _queue_send(self, msg):
         """Queue a message to be sent from the IOLoop's thread.
         
         Parameters
         ----------
         msg : message to send
         
         This is threadsafe, as it uses IOLoop.add_callback to give the loop's
         thread control of the action.
         """
         def thread_send():
             self.session.send(self.stream, msg)
         self.ioloop.add_callback(thread_send)
 
     def _handle_recv(self, msg):
         """callback for stream.on_recv
         
         unpacks message, and calls handlers with it.
         """
         ident,smsg = self.session.feed_identities(msg)
         self.call_handlers(self.session.unserialize(smsg))
     
 
 
 class ShellSocketChannel(ZMQSocketChannel):
     """The XREQ channel for issues request/replies to the kernel.
     """
 
     command_queue = None
     # flag for whether execute requests should be allowed to call raw_input:
     allow_stdin = True
 
     def __init__(self, context, session, address):
         super(ShellSocketChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self.socket = self.context.socket(zmq.DEALER)
         self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         self.socket.connect('tcp://%s:%i' % self.address)
         self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
         self.stream.on_recv(self._handle_recv)
         self._run_loop()
 
     def stop(self):
         self.ioloop.stop()
         super(ShellSocketChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application leve
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
     def execute(self, code, silent=False,
                 user_variables=None, user_expressions=None, allow_stdin=None):
         """Execute code in the kernel.
 
         Parameters
         ----------
         code : str
             A string of Python code.
 
         silent : bool, optional (default False)
             If set, the kernel will execute the code as quietly possible.
 
         user_variables : list, optional
             A list of variable names to pull from the user's namespace.  They
             will come back as a dict with these names as keys and their
             :func:`repr` as values.
 
         user_expressions : dict, optional
             A dict with string keys and  to pull from the user's
             namespace.  They will come back as a dict with these names as keys
             and their :func:`repr` as values.
 
         allow_stdin : bool, optional
             Flag for 
             A dict with string keys and  to pull from the user's
             namespace.  They will come back as a dict with these names as keys
             and their :func:`repr` as values.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         if user_variables is None:
             user_variables = []
         if user_expressions is None:
             user_expressions = {}
         if allow_stdin is None:
             allow_stdin = self.allow_stdin
         
         
         # Don't waste network traffic if inputs are invalid
         if not isinstance(code, basestring):
             raise ValueError('code %r must be a string' % code)
         validate_string_list(user_variables)
         validate_string_dict(user_expressions)
 
         # Create class for content/msg creation. Related to, but possibly
         # not in Session.
         content = dict(code=code, silent=silent,
                        user_variables=user_variables,
                        user_expressions=user_expressions,
                        allow_stdin=allow_stdin,
                        )
         msg = self.session.msg('execute_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def complete(self, text, line, cursor_pos, block=None):
         """Tab complete text in the kernel's namespace.
 
         Parameters
         ----------
         text : str
             The text to complete.
         line : str
             The full line of text that is the surrounding context for the
             text to complete.
         cursor_pos : int
             The position of the cursor in the line where the completion was
             requested.
         block : str, optional
             The full block of code in which the completion is being requested.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         content = dict(text=text, line=line, block=block, cursor_pos=cursor_pos)
         msg = self.session.msg('complete_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def object_info(self, oname):
         """Get metadata information about an object.
 
         Parameters
         ----------
         oname : str
             A string specifying the object name.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         content = dict(oname=oname)
         msg = self.session.msg('object_info_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def history(self, raw=True, output=False, hist_access_type='range', **kwargs):
         """Get entries from the history list.
 
         Parameters
         ----------
         raw : bool
             If True, return the raw input.
         output : bool
             If True, then return the output as well.
         hist_access_type : str
             'range' (fill in session, start and stop params), 'tail' (fill in n)
              or 'search' (fill in pattern param).
 
         session : int
             For a range request, the session from which to get lines. Session
             numbers are positive integers; negative ones count back from the
             current session.
         start : int
             The first line number of a history range.
         stop : int
             The final (excluded) line number of a history range.
 
         n : int
             The number of lines of history to get for a tail request.
 
         pattern : str
             The glob-syntax pattern for a search request.
 
         Returns
         -------
         The msg_id of the message sent.
         """
         content = dict(raw=raw, output=output, hist_access_type=hist_access_type,
                                                                     **kwargs)
         msg = self.session.msg('history_request', content)
         self._queue_send(msg)
         return msg['header']['msg_id']
 
     def shutdown(self, restart=False):
         """Request an immediate kernel shutdown.
 
         Upon receipt of the (empty) reply, client code can safely assume that
         the kernel has shut down and it's safe to forcefully terminate it if
         it's still alive.
 
         The kernel will send the reply via a function registered with Python's
         atexit module, ensuring it's truly done as the kernel is done with all
         normal operation.
         """
         # Send quit message to kernel. Once we implement kernel-side setattr,
         # this should probably be done that way, but for now this will do.
         msg = self.session.msg('shutdown_request', {'restart':restart})
         self._queue_send(msg)
         return msg['header']['msg_id']
 
 
 
 class SubSocketChannel(ZMQSocketChannel):
     """The SUB channel which listens for messages that the kernel publishes.
     """
 
     def __init__(self, context, session, address):
         super(SubSocketChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self.socket = self.context.socket(zmq.SUB)
         self.socket.setsockopt(zmq.SUBSCRIBE,b'')
         self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         self.socket.connect('tcp://%s:%i' % self.address)
         self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
         self.stream.on_recv(self._handle_recv)
         self._run_loop()
 
     def stop(self):
         self.ioloop.stop()
         super(SubSocketChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application leve
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
     def flush(self, timeout=1.0):
         """Immediately processes all pending messages on the SUB channel.
 
         Callers should use this method to ensure that :method:`call_handlers`
         has been called for all messages that have been received on the
         0MQ SUB socket of this channel.
 
         This method is thread safe.
 
         Parameters
         ----------
         timeout : float, optional
             The maximum amount of time to spend flushing, in seconds. The
             default is one second.
         """
         # We do the IOLoop callback process twice to ensure that the IOLoop
         # gets to perform at least one full poll.
         stop_time = time.time() + timeout
         for i in xrange(2):
             self._flushed = False
             self.ioloop.add_callback(self._flush)
             while not self._flushed and time.time() < stop_time:
                 time.sleep(0.01)
 
     def _flush(self):
         """Callback for :method:`self.flush`."""
         self.stream.flush()
         self._flushed = True
 
 
 class StdInSocketChannel(ZMQSocketChannel):
     """A reply channel to handle raw_input requests that the kernel makes."""
 
     msg_queue = None
 
     def __init__(self, context, session, address):
         super(StdInSocketChannel, self).__init__(context, session, address)
         self.ioloop = ioloop.IOLoop()
 
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self.socket = self.context.socket(zmq.DEALER)
         self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
         self.socket.connect('tcp://%s:%i' % self.address)
         self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
         self.stream.on_recv(self._handle_recv)
         self._run_loop()
 
     def stop(self):
         self.ioloop.stop()
         super(StdInSocketChannel, self).stop()
 
     def call_handlers(self, msg):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
         so that some logic must be done to ensure that the application leve
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
     def input(self, string):
         """Send a string of raw input to the kernel."""
         content = dict(value=string)
         msg = self.session.msg('input_reply', content)
         self._queue_send(msg)
 
 
 class HBSocketChannel(ZMQSocketChannel):
     """The heartbeat channel which monitors the kernel heartbeat.
 
     Note that the heartbeat channel is paused by default. As long as you start
     this channel, the kernel manager will ensure that it is paused and un-paused
     as appropriate.
     """
 
     time_to_dead = 3.0
     socket = None
     poller = None
     _running = None
     _pause = None
+    _beating = None
 
     def __init__(self, context, session, address):
         super(HBSocketChannel, self).__init__(context, session, address)
         self._running = False
         self._pause = True
+        self.poller = zmq.Poller()
 
     def _create_socket(self):
+        if self.socket is not None:
+            # close previous socket, before opening a new one
+            self.poller.unregister(self.socket)
+            self.socket.close()
         self.socket = self.context.socket(zmq.REQ)
-        self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
+        self.socket.setsockopt(zmq.LINGER, 0)
         self.socket.connect('tcp://%s:%i' % self.address)
-        self.poller = zmq.Poller()
+        
         self.poller.register(self.socket, zmq.POLLIN)
 
+    def _poll(self, start_time):
+        """poll for heartbeat replies until we reach self.time_to_dead
+        
+        Ignores interrupts, and returns the result of poll(), which
+        will be an empty list if no messages arrived before the timeout,
+        or the event tuple if there is a message to receive.
+        """
+        
+        until_dead = self.time_to_dead - (time.time() - start_time)
+        # ensure poll at least once
+        until_dead = max(until_dead, 1e-3)
+        events = []
+        while True:
+            try:
+                events = self.poller.poll(1000 * until_dead)
+            except zmq.ZMQError as e:
+                if e.errno == errno.EINTR:
+                    # ignore interrupts during heartbeat
+                    # this may never actually happen
+                    until_dead = self.time_to_dead - (time.time() - start_time)
+                    until_dead = max(until_dead, 1e-3)
+                    pass
+                else:
+                    raise
+            else:
+                break
+        return events
+
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self._create_socket()
         self._running = True
+        self._beating = True
+        
         while self._running:
             if self._pause:
+                # just sleep, and skip the rest of the loop
                 time.sleep(self.time_to_dead)
-            else:
+                continue
+            
                 since_last_heartbeat = 0.0
-                request_time = time.time()
-                try:
                     #io.rprint('Ping from HB channel') # dbg
+            # no need to catch EFSM here, because the previous event was
+            # either a recv or connect, which cannot be followed by EFSM
                     self.socket.send(b'ping')
-                except zmq.ZMQError, e:
-                    #io.rprint('*** HB Error:', e) # dbg
-                    if e.errno == zmq.EFSM:
-                        #io.rprint('sleep...', self.time_to_dead) # dbg
-                        time.sleep(self.time_to_dead)
-                        self._create_socket()
-                    else:
-                        raise
-                else:
-                    while True:
-                        try:
-                            self.socket.recv(zmq.NOBLOCK)
-                        except zmq.ZMQError, e:
-                            #io.rprint('*** HB Error 2:', e) # dbg
-                            if e.errno == zmq.EAGAIN:
-                                before_poll = time.time()
-                                until_dead = self.time_to_dead - (before_poll -
-                                                                  request_time)
-
-                                # When the return value of poll() is an empty
-                                # list, that is when things have gone wrong
-                                # (zeromq bug). As long as it is not an empty
-                                # list, poll is working correctly even if it
-                                # returns quickly. Note: poll timeout is in
-                                # milliseconds.
-                                if until_dead > 0.0:
-                                    while True:
-                                        try:
-                                            self.poller.poll(1000 * until_dead)
-                                        except zmq.ZMQError as e:
-                                            if e.errno == errno.EINTR:
+            request_time = time.time()
+            ready = self._poll(request_time)
+            if ready:
+                self._beating = True
+                # the poll above guarantees we have something to recv
+                self.socket.recv()
+                # sleep the remainder of the cycle
+                remainder = self.time_to_dead - (time.time() - request_time)
+                if remainder > 0:
+                    time.sleep(remainder)
                                                 continue
                                             else:
-                                                raise
-                                        else:
-                                            break
-
+                # nothing was received within the time limit, signal heart failure
+                self._beating = False
                                 since_last_heartbeat = time.time()-request_time
-                                if since_last_heartbeat > self.time_to_dead:
                                     self.call_handlers(since_last_heartbeat)
-                                    break
-                            else:
-                                # FIXME: We should probably log this instead.
-                                raise
-                        else:
-                            until_dead = self.time_to_dead - (time.time() -
-                                                              request_time)
-                            if until_dead > 0.0:
-                                #io.rprint('sleep...', self.time_to_dead) # dbg
-                                time.sleep(until_dead)
-                            break
+                # and close/reopen the socket, because the REQ/REP cycle has been broken
+                self._create_socket()
+                continue
 
     def pause(self):
         """Pause the heartbeat."""
         self._pause = True
 
     def unpause(self):
         """Unpause the heartbeat."""
         self._pause = False
 
     def is_beating(self):
-        """Is the heartbeat running and not paused."""
-        if self.is_alive() and not self._pause:
+        """Is the heartbeat running and responsive (and not paused)."""
+        if self.is_alive() and not self._pause and self._beating:
             return True
         else:
             return False
 
     def stop(self):
         self._running = False
         super(HBSocketChannel, self).stop()
 
     def call_handlers(self, since_last_heartbeat):
         """This method is called in the ioloop thread when a message arrives.
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
-        so that some logic must be done to ensure that the application leve
+        so that some logic must be done to ensure that the application level
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
 
 #-----------------------------------------------------------------------------
 # Main kernel manager class
 #-----------------------------------------------------------------------------
 
 class KernelManager(HasTraits):
     """ Manages a kernel for a frontend.
 
     The SUB channel is for the frontend to receive messages published by the
     kernel.
 
     The REQ channel is for the frontend to make requests of the kernel.
 
     The REP channel is for the kernel to request stdin (raw_input) from the
     frontend.
     """
     # config object for passing to child configurables
     config = Instance(Config)
 
     # The PyZMQ Context to use for communication with the kernel.
     context = Instance(zmq.Context)
     def _context_default(self):
         return zmq.Context.instance()
 
     # The Session to use for communication with the kernel.
     session = Instance(Session)
 
     # The kernel process with which the KernelManager is communicating.
     kernel = Instance(Popen)
 
     # The addresses for the communication channels.
     connection_file = Unicode('')
     ip = Unicode(LOCALHOST)
     def _ip_changed(self, name, old, new):
         if new == '*':
             self.ip = '0.0.0.0'
     shell_port = Integer(0)
     iopub_port = Integer(0)
     stdin_port = Integer(0)
     hb_port = Integer(0)
 
     # The classes to use for the various channels.
     shell_channel_class = Type(ShellSocketChannel)
     sub_channel_class = Type(SubSocketChannel)
     stdin_channel_class = Type(StdInSocketChannel)
     hb_channel_class = Type(HBSocketChannel)
 
     # Protected traits.
     _launch_args = Any
     _shell_channel = Any
     _sub_channel = Any
     _stdin_channel = Any
     _hb_channel = Any
     _connection_file_written=Bool(False)
 
     def __init__(self, **kwargs):
         super(KernelManager, self).__init__(**kwargs)
         if self.session is None:
             self.session = Session(config=self.config)
     
     def __del__(self):
-        if self._connection_file_written:
-            # cleanup connection files on full shutdown of kernel we started
-            self._connection_file_written = False
-            try:
-                os.remove(self.connection_file)
-            except IOError:
-                pass
+        self.cleanup_connection_file()
 
 
     #--------------------------------------------------------------------------
     # Channel management methods:
     #--------------------------------------------------------------------------
 
     def start_channels(self, shell=True, sub=True, stdin=True, hb=True):
         """Starts the channels for this kernel.
 
         This will create the channels if they do not exist and then start
         them. If port numbers of 0 are being used (random ports) then you
         must first call :method:`start_kernel`. If the channels have been
         stopped and you call this, :class:`RuntimeError` will be raised.
         """
         if shell:
             self.shell_channel.start()
         if sub:
             self.sub_channel.start()
         if stdin:
             self.stdin_channel.start()
             self.shell_channel.allow_stdin = True
         else:
             self.shell_channel.allow_stdin = False
         if hb:
             self.hb_channel.start()
 
     def stop_channels(self):
         """Stops all the running channels for this kernel.
         """
         if self.shell_channel.is_alive():
             self.shell_channel.stop()
         if self.sub_channel.is_alive():
             self.sub_channel.stop()
         if self.stdin_channel.is_alive():
             self.stdin_channel.stop()
         if self.hb_channel.is_alive():
             self.hb_channel.stop()
 
     @property
     def channels_running(self):
         """Are any of the channels created and running?"""
         return (self.shell_channel.is_alive() or self.sub_channel.is_alive() or
                 self.stdin_channel.is_alive() or self.hb_channel.is_alive())
 
     #--------------------------------------------------------------------------
     # Kernel process management methods:
     #--------------------------------------------------------------------------
     
+    def cleanup_connection_file(self):
+        """cleanup connection file *if we wrote it*
+        
+        Will not raise if the connection file was already removed somehow.
+        """
+        if self._connection_file_written:
+            # cleanup connection files on full shutdown of kernel we started
+            self._connection_file_written = False
+            try:
+                os.remove(self.connection_file)
+            except OSError:
+                pass
+    
     def load_connection_file(self):
         """load connection info from JSON dict in self.connection_file"""
         with open(self.connection_file) as f:
             cfg = json.loads(f.read())
         
         self.ip = cfg['ip']
         self.shell_port = cfg['shell_port']
         self.stdin_port = cfg['stdin_port']
         self.iopub_port = cfg['iopub_port']
         self.hb_port = cfg['hb_port']
         self.session.key = str_to_bytes(cfg['key'])
     
     def write_connection_file(self):
         """write connection info to JSON dict in self.connection_file"""
         if self._connection_file_written:
             return
         self.connection_file,cfg = write_connection_file(self.connection_file,
             ip=self.ip, key=self.session.key,
             stdin_port=self.stdin_port, iopub_port=self.iopub_port,
             shell_port=self.shell_port, hb_port=self.hb_port)
         # write_connection_file also sets default ports:
         self.shell_port = cfg['shell_port']
         self.stdin_port = cfg['stdin_port']
         self.iopub_port = cfg['iopub_port']
         self.hb_port = cfg['hb_port']
         
         self._connection_file_written = True
     
     def start_kernel(self, **kw):
         """Starts a kernel process and configures the manager to use it.
 
         If random ports (port=0) are being used, this method must be called
         before the channels are created.
 
         Parameters:
         -----------
         ipython : bool, optional (default True)
              Whether to use an IPython kernel instead of a plain Python kernel.
 
         launcher : callable, optional (default None)
              A custom function for launching the kernel process (generally a
              wrapper around ``entry_point.base_launch_kernel``). In most cases,
              it should not be necessary to use this parameter.
 
         **kw : optional
              See respective options for IPython and Python kernels.
         """
         if self.ip not in LOCAL_IPS:
             raise RuntimeError("Can only launch a kernel on a local interface. "
                                "Make sure that the '*_address' attributes are "
                                "configured properly. "
                                "Currently valid addresses are: %s"%LOCAL_IPS
                                )
         
         # write connection file / get default ports
         self.write_connection_file()
 
         self._launch_args = kw.copy()
         launch_kernel = kw.pop('launcher', None)
         if launch_kernel is None:
             if kw.pop('ipython', True):
                 from ipkernel import launch_kernel
             else:
                 from pykernel import launch_kernel
         self.kernel = launch_kernel(fname=self.connection_file, **kw)
 
     def shutdown_kernel(self, restart=False):
         """ Attempts to the stop the kernel process cleanly. If the kernel
         cannot be stopped, it is killed, if possible.
         """
         # FIXME: Shutdown does not work on Windows due to ZMQ errors!
         if sys.platform == 'win32':
             self.kill_kernel()
             return
 
         # Pause the heart beat channel if it exists.
         if self._hb_channel is not None:
             self._hb_channel.pause()
 
         # Don't send any additional kernel kill messages immediately, to give
         # the kernel a chance to properly execute shutdown actions. Wait for at
         # most 1s, checking every 0.1s.
         self.shell_channel.shutdown(restart=restart)
         for i in range(10):
             if self.is_alive:
                 time.sleep(0.1)
             else:
                 break
         else:
             # OK, we've waited long enough.
             if self.has_kernel:
                 self.kill_kernel()
 
         if not restart and self._connection_file_written:
             # cleanup connection files on full shutdown of kernel we started
             self._connection_file_written = False
             try:
                 os.remove(self.connection_file)
             except IOError:
                 pass
 
     def restart_kernel(self, now=False, **kw):
         """Restarts a kernel with the arguments that were used to launch it.
 
         If the old kernel was launched with random ports, the same ports will be
         used for the new kernel.
 
         Parameters
         ----------
         now : bool, optional
             If True, the kernel is forcefully restarted *immediately*, without
             having a chance to do any cleanup action.  Otherwise the kernel is
             given 1s to clean up before a forceful restart is issued.
 
             In all cases the kernel is restarted, the only difference is whether
             it is given a chance to perform a clean shutdown or not.
 
         **kw : optional
             Any options specified here will replace those used to launch the
             kernel.
         """
         if self._launch_args is None:
             raise RuntimeError("Cannot restart the kernel. "
                                "No previous call to 'start_kernel'.")
         else:
             # Stop currently running kernel.
             if self.has_kernel:
                 if now:
                     self.kill_kernel()
                 else:
                     self.shutdown_kernel(restart=True)
 
             # Start new kernel.
             self._launch_args.update(kw)
             self.start_kernel(**self._launch_args)
 
             # FIXME: Messages get dropped in Windows due to probable ZMQ bug
             # unless there is some delay here.
             if sys.platform == 'win32':
                 time.sleep(0.2)
 
     @property
     def has_kernel(self):
         """Returns whether a kernel process has been specified for the kernel
         manager.
         """
         return self.kernel is not None
 
     def kill_kernel(self):
         """ Kill the running kernel. """
         if self.has_kernel:
             # Pause the heart beat channel if it exists.
             if self._hb_channel is not None:
                 self._hb_channel.pause()
 
             # Attempt to kill the kernel.
             try:
                 self.kernel.kill()
             except OSError, e:
                 # In Windows, we will get an Access Denied error if the process
                 # has already terminated. Ignore it.
                 if sys.platform == 'win32':
                     if e.winerror != 5:
                         raise
                 # On Unix, we may get an ESRCH error if the process has already
                 # terminated. Ignore it.
                 else:
                     from errno import ESRCH
                     if e.errno != ESRCH:
                         raise
             self.kernel = None
         else:
             raise RuntimeError("Cannot kill kernel. No kernel is running!")
 
     def interrupt_kernel(self):
         """ Interrupts the kernel. Unlike ``signal_kernel``, this operation is
         well supported on all platforms.
         """
         if self.has_kernel:
             if sys.platform == 'win32':
                 from parentpoller import ParentPollerWindows as Poller
                 Poller.send_interrupt(self.kernel.win32_interrupt_event)
             else:
                 self.kernel.send_signal(signal.SIGINT)
         else:
             raise RuntimeError("Cannot interrupt kernel. No kernel is running!")
 
     def signal_kernel(self, signum):
         """ Sends a signal to the kernel. Note that since only SIGTERM is
         supported on Windows, this function is only useful on Unix systems.
         """
         if self.has_kernel:
             self.kernel.send_signal(signum)
         else:
             raise RuntimeError("Cannot signal kernel. No kernel is running!")
 
     @property
     def is_alive(self):
         """Is the kernel process still running?"""
-        # FIXME: not using a heartbeat means this method is broken for any
-        # remote kernel, it's only capable of handling local kernels.
         if self.has_kernel:
             if self.kernel.poll() is None:
                 return True
             else:
                 return False
+        elif self._hb_channel is not None:
+            # We didn't start the kernel with this KernelManager so we
+            # use the heartbeat.
+            return self._hb_channel.is_beating()
         else:
-            # We didn't start the kernel with this KernelManager so we don't
-            # know if it is running. We should use a heartbeat for this case.
+            # no heartbeat and not local, we can't tell if it's running,
+            # so naively return True
             return True
 
     #--------------------------------------------------------------------------
     # Channels used for communication with the kernel:
     #--------------------------------------------------------------------------
 
     @property
     def shell_channel(self):
         """Get the REQ socket channel object to make requests of the kernel."""
         if self._shell_channel is None:
             self._shell_channel = self.shell_channel_class(self.context,
                                                          self.session,
                                                          (self.ip, self.shell_port))
         return self._shell_channel
 
     @property
     def sub_channel(self):
         """Get the SUB socket channel object."""
         if self._sub_channel is None:
             self._sub_channel = self.sub_channel_class(self.context,
                                                        self.session,
                                                        (self.ip, self.iopub_port))
         return self._sub_channel
 
     @property
     def stdin_channel(self):
         """Get the REP socket channel object to handle stdin (raw_input)."""
         if self._stdin_channel is None:
             self._stdin_channel = self.stdin_channel_class(self.context,
                                                        self.session,
                                                        (self.ip, self.stdin_port))
         return self._stdin_channel
 
     @property
     def hb_channel(self):
         """Get the heartbeat socket channel object to check that the
         kernel is alive."""
         if self._hb_channel is None:
             self._hb_channel = self.hb_channel_class(self.context,
                                                        self.session,
                                                        (self.ip, self.hb_port))
         return self._hb_channel

IPython/zmq/zmqshell.py

@@ -1,512 +1,509
 """A ZMQ-based subclass of InteractiveShell.
 
 This code is meant to ease the refactoring of the base InteractiveShell into
 something with a cleaner architecture for 2-process use, without actually
 breaking InteractiveShell itself.  So we're doing something a bit ugly, where
 we subclass and override what we want to fix.  Once this is working well, we
 can go back to the base class and refactor the code for a cleaner inheritance
 implementation that doesn't rely on so much monkeypatching.
 
 But this lets us maintain a fully working IPython as we develop the new
 machinery.  This should thus be thought of as scaffolding.
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Stdlib
 import inspect
 import os
 import sys
 from subprocess import Popen, PIPE
 
 # Our own
 from IPython.core.interactiveshell import (
     InteractiveShell, InteractiveShellABC
 )
 from IPython.core import page, pylabtools
 from IPython.core.autocall import ZMQExitAutocall
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.macro import Macro
 from IPython.core.magic import MacroToEdit
 from IPython.core.payloadpage import install_payload_page
 from IPython.lib.kernel import (
     get_connection_file, get_connection_info, connect_qtconsole
 )
 from IPython.utils import io
 from IPython.utils.jsonutil import json_clean
 from IPython.utils.path import get_py_filename
 from IPython.utils.process import arg_split
 from IPython.utils.traitlets import Instance, Type, Dict, CBool
 from IPython.utils.warn import warn, error
 from IPython.zmq.displayhook import ZMQShellDisplayHook, _encode_binary
 from IPython.zmq.session import extract_header
 from session import Session
 
-#-----------------------------------------------------------------------------
-# Globals and side-effects
-#-----------------------------------------------------------------------------
-
-# Install the payload version of page.
-install_payload_page()
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 class ZMQDisplayPublisher(DisplayPublisher):
     """A display publisher that publishes data using a ZeroMQ PUB socket."""
 
     session = Instance(Session)
     pub_socket = Instance('zmq.Socket')
     parent_header = Dict({})
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""
         self.parent_header = extract_header(parent)
 
     def publish(self, source, data, metadata=None):
         if metadata is None:
             metadata = {}
         self._validate_data(source, data, metadata)
         content = {}
         content['source'] = source
         _encode_binary(data)
         content['data'] = data
         content['metadata'] = metadata
         self.session.send(
             self.pub_socket, u'display_data', json_clean(content),
             parent=self.parent_header
         )
 
     def clear_output(self, stdout=True, stderr=True, other=True):
         content = dict(stdout=stdout, stderr=stderr, other=other)
         self.session.send(
             self.pub_socket, u'clear_output', content,
             parent=self.parent_header
         )
 
 class ZMQInteractiveShell(InteractiveShell):
     """A subclass of InteractiveShell for ZMQ."""
 
     displayhook_class = Type(ZMQShellDisplayHook)
     display_pub_class = Type(ZMQDisplayPublisher)
 
     # Override the traitlet in the parent class, because there's no point using
     # readline for the kernel. Can be removed when the readline code is moved
     # to the terminal frontend.
     colors_force = CBool(True)
     readline_use = CBool(False)
     # autoindent has no meaning in a zmqshell, and attempting to enable it
     # will print a warning in the absence of readline.
     autoindent = CBool(False)
 
     exiter = Instance(ZMQExitAutocall)
     def _exiter_default(self):
         return ZMQExitAutocall(self)
 
     keepkernel_on_exit = None
 
     # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
     # interactive input being read; we provide event loop support in ipkernel
     from .eventloops import enable_gui
     enable_gui = staticmethod(enable_gui)
 
     def init_environment(self):
         """Configure the user's environment.
 
         """
         env = os.environ
         # These two ensure 'ls' produces nice coloring on BSD-derived systems
         env['TERM'] = 'xterm-color'
         env['CLICOLOR'] = '1'
         # Since normal pagers don't work at all (over pexpect we don't have
         # single-key control of the subprocess), try to disable paging in
         # subprocesses as much as possible.
         env['PAGER'] = 'cat'
         env['GIT_PAGER'] = 'cat'
 
+        # And install the payload version of page.
+        install_payload_page()
+
     def auto_rewrite_input(self, cmd):
         """Called to show the auto-rewritten input for autocall and friends.
 
         FIXME: this payload is currently not correctly processed by the
         frontend.
         """
         new = self.prompt_manager.render('rewrite') + cmd
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.auto_rewrite_input',
             transformed_input=new,
             )
         self.payload_manager.write_payload(payload)
 
     def ask_exit(self):
         """Engage the exit actions."""
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.ask_exit',
             exit=True,
             keepkernel=self.keepkernel_on_exit,
             )
         self.payload_manager.write_payload(payload)
 
     def _showtraceback(self, etype, evalue, stb):
 
         exc_content = {
             u'traceback' : stb,
             u'ename' : unicode(etype.__name__),
             u'evalue' : unicode(evalue)
         }
 
         dh = self.displayhook
         # Send exception info over pub socket for other clients than the caller
         # to pick up
         exc_msg = dh.session.send(dh.pub_socket, u'pyerr', json_clean(exc_content), dh.parent_header)
 
         # FIXME - Hack: store exception info in shell object.  Right now, the
         # caller is reading this info after the fact, we need to fix this logic
         # to remove this hack.  Even uglier, we need to store the error status
         # here, because in the main loop, the logic that sets it is being
         # skipped because runlines swallows the exceptions.
         exc_content[u'status'] = u'error'
         self._reply_content = exc_content
         # /FIXME
 
         return exc_content
 
     #------------------------------------------------------------------------
     # Magic overrides
     #------------------------------------------------------------------------
     # Once the base class stops inheriting from magic, this code needs to be
     # moved into a separate machinery as well.  For now, at least isolate here
     # the magics which this class needs to implement differently from the base
     # class, or that are unique to it.
 
     def magic_doctest_mode(self,parameter_s=''):
         """Toggle doctest mode on and off.
 
         This mode is intended to make IPython behave as much as possible like a
         plain Python shell, from the perspective of how its prompts, exceptions
         and output look.  This makes it easy to copy and paste parts of a
         session into doctests.  It does so by:
 
         - Changing the prompts to the classic ``>>>`` ones.
         - Changing the exception reporting mode to 'Plain'.
         - Disabling pretty-printing of output.
 
         Note that IPython also supports the pasting of code snippets that have
         leading '>>>' and '...' prompts in them.  This means that you can paste
         doctests from files or docstrings (even if they have leading
         whitespace), and the code will execute correctly.  You can then use
         '%history -t' to see the translated history; this will give you the
         input after removal of all the leading prompts and whitespace, which
         can be pasted back into an editor.
 
         With these features, you can switch into this mode easily whenever you
         need to do testing and changes to doctests, without having to leave
         your existing IPython session.
         """
 
         from IPython.utils.ipstruct import Struct
 
         # Shorthands
         shell = self.shell
         disp_formatter = self.shell.display_formatter
         ptformatter = disp_formatter.formatters['text/plain']
         # dstore is a data store kept in the instance metadata bag to track any
         # changes we make, so we can undo them later.
         dstore = shell.meta.setdefault('doctest_mode', Struct())
         save_dstore = dstore.setdefault
 
         # save a few values we'll need to recover later
         mode = save_dstore('mode', False)
         save_dstore('rc_pprint', ptformatter.pprint)
         save_dstore('rc_plain_text_only',disp_formatter.plain_text_only)
         save_dstore('xmode', shell.InteractiveTB.mode)
 
         if mode == False:
             # turn on
             ptformatter.pprint = False
             disp_formatter.plain_text_only = True
             shell.magic_xmode('Plain')
         else:
             # turn off
             ptformatter.pprint = dstore.rc_pprint
             disp_formatter.plain_text_only = dstore.rc_plain_text_only
             shell.magic_xmode(dstore.xmode)
 
         # Store new mode and inform on console
         dstore.mode = bool(1-int(mode))
         mode_label = ['OFF','ON'][dstore.mode]
         print('Doctest mode is:', mode_label)
 
         # Send the payload back so that clients can modify their prompt display
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.magic_doctest_mode',
             mode=dstore.mode)
         self.payload_manager.write_payload(payload)
 
     def magic_edit(self,parameter_s='',last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs an external text editor. You will need to set the command for
         this editor via the ``TerminalInteractiveShell.editor`` option in your
         configuration file before it will work.
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
 
         Options:
 
         -n <number>: open the editor at a specified line number.  By default,
         the IPython editor hook uses the unix syntax 'editor +N filename', but
         you can configure this by providing your own modified hook if your
         favorite editor supports line-number specifications with a different
         syntax.
 
         -p: this will call the editor with the same data as the previous time
         it was used, regardless of how long ago (in your current session) it
         was.
 
         -r: use 'raw' input.  This option only applies to input taken from the
         user's history.  By default, the 'processed' history is used, so that
         magics are loaded in their transformed version to valid Python.  If
         this option is given, the raw input as typed as the command line is
         used instead.  When you exit the editor, it will be executed by
         IPython's own processor.
 
         -x: do not execute the edited code immediately upon exit. This is
         mainly useful if you are editing programs which need to be called with
         command line arguments, which you can then do using %run.
 
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
         1 4:8 9). These are interpreted as lines of previous input to be
         loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
         variable and its contents loaded into the editor. You can thus edit
         any string which contains python code (including the result of
         previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
         editor at the point where it is defined. You can use `%edit function`
         to load an editor exactly at the point where 'function' is defined,
         edit it and have the file be executed automatically.
 
         If the object is a macro (see %macro for details), this opens up your
         specified editor with a temporary file containing the macro's data.
         Upon exit, the macro is reloaded with the contents of the file.
 
         Note: opening at an exact line is only supported under Unix, and some
         editors (like kedit and gedit up to Gnome 2.8) do not understand the
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
         file with that name (adding .py if necessary) and load it into the
         editor. It will execute its contents with execfile() when you exit,
         loading any code in the file into your interactive namespace.
 
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
         via _<NUMBER> or Out[<NUMBER>], where <NUMBER> is the prompt number of
         the output.
 
         Note that %edit is also available through the alias %ed.
 
         This is an example of creating a simple function inside the editor and
         then modifying it. First, start up the editor:
 
         In [1]: ed
         Editing... done. Executing edited code...
         Out[1]: 'def foo():n    print "foo() was defined in an editing session"n'
 
         We can then call the function foo():
 
         In [2]: foo()
         foo() was defined in an editing session
 
         Now we edit foo.  IPython automatically loads the editor with the
         (temporary) file where foo() was previously defined:
 
         In [3]: ed foo
         Editing... done. Executing edited code...
 
         And if we call foo() again we get the modified version:
 
         In [4]: foo()
         foo() has now been changed!
 
         Here is an example of how to edit a code snippet successive
         times. First we call the editor:
 
         In [5]: ed
         Editing... done. Executing edited code...
         hello
         Out[5]: "print 'hello'n"
 
         Now we call it again with the previous output (stored in _):
 
         In [6]: ed _
         Editing... done. Executing edited code...
         hello world
         Out[6]: "print 'hello world'n"
 
         Now we call it with the output #8 (stored in _8, also as Out[8]):
 
         In [7]: ed _8
         Editing... done. Executing edited code...
         hello again
         Out[7]: "print 'hello again'n"
         """
 
         opts,args = self.parse_options(parameter_s,'prn:')
 
         try:
             filename, lineno, _ = self._find_edit_target(args, opts, last_call)
         except MacroToEdit as e:
             # TODO: Implement macro editing over 2 processes.
             print("Macro editing not yet implemented in 2-process model.")
             return
 
         # Make sure we send to the client an absolute path, in case the working
         # directory of client and kernel don't match
         filename = os.path.abspath(filename)
 
         payload = {
             'source' : 'IPython.zmq.zmqshell.ZMQInteractiveShell.edit_magic',
             'filename' : filename,
             'line_number' : lineno
         }
         self.payload_manager.write_payload(payload)
 
     # A few magics that are adapted to the specifics of using pexpect and a
     # remote terminal
 
     def magic_clear(self, arg_s):
         """Clear the terminal."""
         if os.name == 'posix':
             self.shell.system("clear")
         else:
             self.shell.system("cls")
 
     if os.name == 'nt':
         # This is the usual name in windows
         magic_cls = magic_clear
 
     # Terminal pagers won't work over pexpect, but we do have our own pager
 
     def magic_less(self, arg_s):
         """Show a file through the pager.
 
         Files ending in .py are syntax-highlighted."""
         cont = open(arg_s).read()
         if arg_s.endswith('.py'):
             cont = self.shell.pycolorize(cont)
         page.page(cont)
 
     magic_more = magic_less
 
     # Man calls a pager, so we also need to redefine it
     if os.name == 'posix':
         def magic_man(self, arg_s):
             """Find the man page for the given command and display in pager."""
             page.page(self.shell.getoutput('man %s | col -b' % arg_s,
                                            split=False))
 
     # FIXME: this is specific to the GUI, so we should let the gui app load
     # magics at startup that are only for the gui.  Once the gui app has proper
     # profile and configuration management, we can have it initialize a kernel
     # with a special config file that provides these.
     def magic_guiref(self, arg_s):
         """Show a basic reference about the GUI console."""
         from IPython.core.usage import gui_reference
         page.page(gui_reference, auto_html=True)
     
     def magic_connect_info(self, arg_s):
         """Print information for connecting other clients to this kernel
         
         It will print the contents of this session's connection file, as well as
         shortcuts for local clients.
         
         In the simplest case, when called from the most recently launched kernel,
         secondary clients can be connected, simply with:
         
         $> ipython <app> --existing
         
         """
         
         from IPython.core.application import BaseIPythonApplication as BaseIPApp
         
         if BaseIPApp.initialized():
             app = BaseIPApp.instance()
             security_dir = app.profile_dir.security_dir
             profile = app.profile
         else:
             profile = 'default'
             security_dir = ''
         
         try:
             connection_file = get_connection_file()
             info = get_connection_info(unpack=False)
         except Exception as e:
             error("Could not get connection info: %r" % e)
             return
         
         # add profile flag for non-default profile
         profile_flag = "--profile %s" % profile if profile != 'default' else ""
         
         # if it's in the security dir, truncate to basename
         if security_dir == os.path.dirname(connection_file):
             connection_file = os.path.basename(connection_file)
         
         
         print (info + '\n')
         print ("Paste the above JSON into a file, and connect with:\n"
             "    $> ipython <app> --existing <file>\n"
             "or, if you are local, you can connect with just:\n"
             "    $> ipython <app> --existing {0} {1}\n"
             "or even just:\n"
             "    $> ipython <app> --existing {1}\n"
             "if this is the most recent IPython session you have started.".format(
             connection_file, profile_flag
             )
         )
 
     def magic_qtconsole(self, arg_s):
         """Open a qtconsole connected to this kernel.
         
         Useful for connecting a qtconsole to running notebooks, for better
         debugging.
         """
         try:
             p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
         except Exception as e:
             error("Could not start qtconsole: %r" % e)
             return
 
     def set_next_input(self, text):
         """Send the specified text to the frontend to be presented at the next
         input cell."""
         payload = dict(
             source='IPython.zmq.zmqshell.ZMQInteractiveShell.set_next_input',
             text=text
         )
         self.payload_manager.write_payload(payload)
 
 
 InteractiveShellABC.register(ZMQInteractiveShell)