Merge pull request #864 from ipython/termzmq

Two-process terminal frontend: this branch adds a new IPython frontend, invoked via

ipython console

that behaves much like the regular, old ipython, but runs over zeromq in two processes.  This means that such a client can connect to existing kernels initiated by the Qt console, the notebook or standalone (i.e. via `ipython kernel`).

We still have some internal architectural cleanups to perform to simplify how the various frontends talk to the kernels, but by having this main piece in, the complete picture is clearer, and that refactoring work can be carried post-0.12.

This frontend should still be considered experimental.

IPython/frontend/consoleapp.py

@@ -0,0 +1,352 b''
+""" A minimal application base mixin for all ZMQ based IPython frontends.
+
+This is not a complete console app, as subprocess will not be able to receive
+input, there is no real readline support, among other limitations. This is a
+refactoring of what used to be the IPython/frontend/qt/console/qtconsoleapp.py
+
+Authors:
+
+* Evan Patterson
+* Min RK
+* Erik Tollerud
+* Fernando Perez
+* Bussonnier Matthias
+* Thomas Kluyver
+* Paul Ivanov
+
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# stdlib imports
+import atexit
+import json
+import os
+import signal
+import sys
+import uuid
+
+
+# Local imports
+from IPython.config.application import boolean_flag
+from IPython.config.configurable import Configurable
+from IPython.core.profiledir import ProfileDir
+from IPython.lib.kernel import tunnel_to_kernel, find_connection_file, swallow_argv
+from IPython.zmq.blockingkernelmanager import BlockingKernelManager
+from IPython.utils.path import filefind
+from IPython.utils.py3compat import str_to_bytes
+from IPython.utils.traitlets import (
+    Dict, List, Unicode, CUnicode, Int, CBool, Any
+)
+from IPython.zmq.ipkernel import (
+    flags as ipkernel_flags,
+    aliases as ipkernel_aliases,
+    IPKernelApp
+)
+from IPython.zmq.session import Session, default_secure
+from IPython.zmq.zmqshell import ZMQInteractiveShell
+
+#-----------------------------------------------------------------------------
+# Network Constants
+#-----------------------------------------------------------------------------
+
+from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+
+#-----------------------------------------------------------------------------
+# Aliases and Flags
+#-----------------------------------------------------------------------------
+
+flags = dict(ipkernel_flags)
+
+# the flags that are specific to the frontend
+# these must be scrubbed before being passed to the kernel,
+# or it will raise an error on unrecognized flags
+app_flags = {
+    'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}},
+            "Connect to an existing kernel. If no argument specified, guess most recent"),
+}
+app_flags.update(boolean_flag(
+    'confirm-exit', 'IPythonConsoleApp.confirm_exit',
+    """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
+       to force a direct exit without any confirmation.
+    """,
+    """Don't prompt the user when exiting. This will terminate the kernel
+       if it is owned by the frontend, and leave it alive if it is external.
+    """
+))
+flags.update(app_flags)
+
+aliases = dict(ipkernel_aliases)
+
+# also scrub aliases from the frontend
+app_aliases = dict(
+    hb = 'IPythonConsoleApp.hb_port',
+    shell = 'IPythonConsoleApp.shell_port',
+    iopub = 'IPythonConsoleApp.iopub_port',
+    stdin = 'IPythonConsoleApp.stdin_port',
+    ip = 'IPythonConsoleApp.ip',
+    existing = 'IPythonConsoleApp.existing',
+    f = 'IPythonConsoleApp.connection_file',
+
+
+    ssh = 'IPythonConsoleApp.sshserver',
+)
+aliases.update(app_aliases)
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# IPythonConsole
+#-----------------------------------------------------------------------------
+
+
+class IPythonConsoleApp(Configurable):
+    name = 'ipython-console-mixin'
+    default_config_file_name='ipython_config.py'
+
+    description = """
+        The IPython Mixin Console.
+        
+        This class contains the common portions of console client (QtConsole,
+        ZMQ-based terminal console, etc).  It is not a full console, in that
+        launched terminal subprocesses will not be able to accept input.
+        
+        The Console using this mixing supports various extra features beyond
+        the single-process Terminal IPython shell, such as connecting to
+        existing kernel, via:
+        
+            ipython <appname> --existing
+        
+        as well as tunnel via SSH
+        
+    """
+
+    classes = [IPKernelApp, ZMQInteractiveShell, ProfileDir, Session]
+    flags = Dict(flags)
+    aliases = Dict(aliases)
+    kernel_manager_class = BlockingKernelManager
+
+    kernel_argv = List(Unicode)
+    # frontend flags&aliases to be stripped when building kernel_argv
+    frontend_flags = Any(app_flags)
+    frontend_aliases = Any(app_aliases)
+
+    pure = CBool(False, config=True,
+        help="Use a pure Python kernel instead of an IPython kernel.")
+    # create requested profiles by default, if they don't exist:
+    auto_create = CBool(True)
+    # connection info:
+    ip = Unicode(LOCALHOST, config=True,
+        help="""Set the kernel\'s IP address [default localhost].
+        If the IP address is something other than localhost, then
+        Consoles on other machines will be able to connect
+        to the Kernel, so be careful!"""
+    )
+    
+    sshserver = Unicode('', config=True,
+        help="""The SSH server to use to connect to the kernel.""")
+    sshkey = Unicode('', config=True,
+        help="""Path to the ssh key to use for logging in to the ssh server.""")
+    
+    hb_port = Int(0, config=True,
+        help="set the heartbeat port [default: random]")
+    shell_port = Int(0, config=True,
+        help="set the shell (XREP) port [default: random]")
+    iopub_port = Int(0, config=True,
+        help="set the iopub (PUB) port [default: random]")
+    stdin_port = Int(0, config=True,
+        help="set the stdin (XREQ) port [default: random]")
+    connection_file = Unicode('', config=True,
+        help="""JSON file in which to store connection info [default: kernel-<pid>.json]
+
+        This file will contain the IP, ports, and authentication key needed to connect
+        clients to this kernel. By default, this file will be created in the security-dir
+        of the current profile, but can be specified by absolute path.
+        """)
+    def _connection_file_default(self):
+        return 'kernel-%i.json' % os.getpid()
+
+    existing = CUnicode('', config=True,
+        help="""Connect to an already running kernel""")
+
+    confirm_exit = CBool(True, config=True,
+        help="""
+        Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
+        to force a direct exit without any confirmation.""",
+    )
+
+
+    def build_kernel_argv(self, argv=None):
+        """build argv to be passed to kernel subprocess"""
+        if argv is None:
+            argv = sys.argv[1:]
+        self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags)
+        # kernel should inherit default config file from frontend
+        self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
+    
+    def init_connection_file(self):
+        """find the connection file, and load the info if found.
+        
+        The current working directory and the current profile's security
+        directory will be searched for the file if it is not given by
+        absolute path.
+        
+        When attempting to connect to an existing kernel and the `--existing`
+        argument does not match an existing file, it will be interpreted as a
+        fileglob, and the matching file in the current profile's security dir
+        with the latest access time will be used.
+        
+        After this method is called, self.connection_file contains the *full path*
+        to the connection file, never just its name.
+        """
+        if self.existing:
+            try:
+                cf = find_connection_file(self.existing)
+            except Exception:
+                self.log.critical("Could not find existing kernel connection file %s", self.existing)
+                self.exit(1)
+            self.log.info("Connecting to existing kernel: %s" % cf)
+            self.connection_file = cf
+        else:
+            # not existing, check if we are going to write the file
+            # and ensure that self.connection_file is a full path, not just the shortname
+            try:
+                cf = find_connection_file(self.connection_file)
+            except Exception:
+                # file might not exist
+                if self.connection_file == os.path.basename(self.connection_file):
+                    # just shortname, put it in security dir
+                    cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
+                else:
+                    cf = self.connection_file
+                self.connection_file = cf
+        
+        # should load_connection_file only be used for existing?
+        # as it is now, this allows reusing ports if an existing
+        # file is requested
+        try:
+            self.load_connection_file()
+        except Exception:
+            self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
+            self.exit(1)
+    
+    def load_connection_file(self):
+        """load ip/port/hmac config from JSON connection file"""
+        # this is identical to KernelApp.load_connection_file
+        # perhaps it can be centralized somewhere?
+        try:
+            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+        except IOError:
+            self.log.debug("Connection File not found: %s", self.connection_file)
+            return
+        self.log.debug(u"Loading connection file %s", fname)
+        with open(fname) as f:
+            s = f.read()
+        cfg = json.loads(s)
+        if self.ip == LOCALHOST and 'ip' in cfg:
+            # not overridden by config or cl_args
+            self.ip = cfg['ip']
+        for channel in ('hb', 'shell', 'iopub', 'stdin'):
+            name = channel + '_port'
+            if getattr(self, name) == 0 and name in cfg:
+                # not overridden by config or cl_args
+                setattr(self, name, cfg[name])
+        if 'key' in cfg:
+            self.config.Session.key = str_to_bytes(cfg['key'])
+    
+    def init_ssh(self):
+        """set up ssh tunnels, if needed."""
+        if not self.sshserver and not self.sshkey:
+            return
+        
+        if self.sshkey and not self.sshserver:
+            # specifying just the key implies that we are connecting directly
+            self.sshserver = self.ip
+            self.ip = LOCALHOST
+        
+        # build connection dict for tunnels:
+        info = dict(ip=self.ip,
+                    shell_port=self.shell_port,
+                    iopub_port=self.iopub_port,
+                    stdin_port=self.stdin_port,
+                    hb_port=self.hb_port
+        )
+        
+        self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
+        
+        # tunnels return a new set of ports, which will be on localhost:
+        self.ip = LOCALHOST
+        try:
+            newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
+        except:
+            # even catch KeyboardInterrupt
+            self.log.error("Could not setup tunnels", exc_info=True)
+            self.exit(1)
+        
+        self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
+        
+        cf = self.connection_file
+        base,ext = os.path.splitext(cf)
+        base = os.path.basename(base)
+        self.connection_file = os.path.basename(base)+'-ssh'+ext
+        self.log.critical("To connect another client via this tunnel, use:")
+        self.log.critical("--existing %s" % self.connection_file)
+    
+    def _new_connection_file(self):
+        cf = ''
+        while not cf:
+            # we don't need a 128b id to distinguish kernels, use more readable
+            # 48b node segment (12 hex chars).  Users running more than 32k simultaneous
+            # kernels can subclass.
+            ident = str(uuid.uuid4()).split('-')[-1]
+            cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident)
+            # only keep if it's actually new.  Protect against unlikely collision
+            # in 48b random search space
+            cf = cf if not os.path.exists(cf) else ''
+        return cf
+
+    def init_kernel_manager(self):
+        # Don't let Qt or ZMQ swallow KeyboardInterupts.
+        signal.signal(signal.SIGINT, signal.SIG_DFL)
+
+        # Create a KernelManager and start a kernel.
+        self.kernel_manager = self.kernel_manager_class(
+                                ip=self.ip,
+                                shell_port=self.shell_port,
+                                iopub_port=self.iopub_port,
+                                stdin_port=self.stdin_port,
+                                hb_port=self.hb_port,
+                                connection_file=self.connection_file,
+                                config=self.config,
+        )
+        # start the kernel
+        if not self.existing:
+            kwargs = dict(ipython=not self.pure)
+            kwargs['extra_arguments'] = self.kernel_argv
+            self.kernel_manager.start_kernel(**kwargs)
+        elif self.sshserver:
+            # ssh, write new connection file
+            self.kernel_manager.write_connection_file()
+        atexit.register(self.kernel_manager.cleanup_connection_file)
+        self.kernel_manager.start_channels()
+
+
+    def initialize(self, argv=None):
+        """
+        Classes which mix this class in should call:
+               IPythonConsoleApp.initialize(self,argv)
+        """
+        self.init_connection_file()
+        default_secure(self.config)
+        self.init_ssh()
+        self.init_kernel_manager()
+

IPython/frontend/terminal/console/app.py

@@ -0,0 +1,152 b''
+""" A minimal application using the ZMQ-based terminal IPython frontend.
+
+This is not a complete console app, as subprocess will not be able to receive
+input, there is no real readline support, among other limitations.
+
+Authors:
+
+* Min RK
+* Paul Ivanov
+
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+import signal
+import sys
+import time
+
+from IPython.frontend.terminal.ipapp import TerminalIPythonApp, frontend_flags as term_flags
+
+from IPython.utils.traitlets import (
+    Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
+)
+from IPython.utils.warn import warn,error
+
+from IPython.zmq.ipkernel import IPKernelApp
+from IPython.zmq.session import Session, default_secure
+from IPython.zmq.zmqshell import ZMQInteractiveShell
+from IPython.frontend.consoleapp import (
+        IPythonConsoleApp, app_aliases, app_flags, aliases, app_aliases, flags
+    )
+
+from IPython.frontend.terminal.console.interactiveshell import ZMQTerminalInteractiveShell
+
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
+
+_examples = """
+ipython console # start the ZMQ-based console
+ipython console --existing # connect to an existing ipython session
+"""
+
+#-----------------------------------------------------------------------------
+# Flags and Aliases
+#-----------------------------------------------------------------------------
+
+# copy flags from mixin:
+flags = dict(flags)
+# start with mixin frontend flags:
+frontend_flags = dict(app_flags)
+# add TerminalIPApp flags:
+frontend_flags.update(term_flags)
+# pylab is not frontend-specific in two-process IPython
+frontend_flags.pop('pylab')
+# disable quick startup, as it won't propagate to the kernel anyway
+frontend_flags.pop('quick')
+# update full dict with frontend flags:
+flags.update(frontend_flags)
+
+# copy flags from mixin
+aliases = dict(aliases)
+# start with mixin frontend flags
+frontend_aliases = dict(app_aliases)
+# load updated frontend flags into full dict
+aliases.update(frontend_aliases)
+
+# get flags&aliases into sets, and remove a couple that
+# shouldn't be scrubbed from backend flags:
+frontend_aliases = set(frontend_aliases.keys())
+frontend_flags = set(frontend_flags.keys())
+
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class ZMQTerminalIPythonApp(TerminalIPythonApp, IPythonConsoleApp):
+    name = "ipython-console"
+    """Start a terminal frontend to the IPython zmq kernel."""
+
+    description = """
+        The IPython terminal-based Console.
+
+        This launches a Console application inside a terminal.
+
+        The Console supports various extra features beyond the traditional
+        single-process Terminal IPython shell, such as connecting to an
+        existing ipython session, via:
+
+            ipython console --existing
+
+        where the previous session could have been created by another ipython
+        console, an ipython qtconsole, or by opening an ipython notebook.
+
+    """
+    examples = _examples
+
+    classes = List([IPKernelApp, ZMQTerminalInteractiveShell, Session])
+    flags = Dict(flags)
+    aliases = Dict(aliases)
+    frontend_aliases = Any(frontend_aliases)
+    frontend_flags = Any(frontend_flags)
+    
+    subcommands = Dict()
+
+    def parse_command_line(self, argv=None):
+        super(ZMQTerminalIPythonApp, self).parse_command_line(argv)
+        self.build_kernel_argv(argv)
+
+    def init_shell(self):
+        IPythonConsoleApp.initialize(self)
+        # relay sigint to kernel
+        signal.signal(signal.SIGINT, self.handle_sigint)
+        self.shell = ZMQTerminalInteractiveShell.instance(config=self.config,
+                        display_banner=False, profile_dir=self.profile_dir,
+                        ipython_dir=self.ipython_dir, kernel_manager=self.kernel_manager)
+
+    def handle_sigint(self, *args):
+        if self.shell._executing:
+            if self.kernel_manager.has_kernel:
+                # interrupt already gets passed to subprocess by signal handler.
+                # Only if we prevent that should we need to explicitly call
+                # interrupt_kernel, until which time, this would result in a 
+                # double-interrupt:
+                # self.kernel_manager.interrupt_kernel()
+                pass
+            else:
+                self.shell.write_err('\n')
+                error("Cannot interrupt kernels we didn't start.\n")
+        else:
+            # raise the KeyboardInterrupt if we aren't waiting for execution,
+            # so that the interact loop advances, and prompt is redrawn, etc.
+            raise KeyboardInterrupt
+            
+
+    def init_code(self):
+        # no-op in the frontend, code gets run in the backend
+        pass
+
+def launch_new_instance():
+    """Create and run a full blown IPython instance"""
+    app = ZMQTerminalIPythonApp.instance()
+    app.initialize()
+    app.start()
+
+
+if __name__ == '__main__':
+    launch_new_instance()
+

IPython/frontend/terminal/console/completer.py

@@ -0,0 +1,44 b''
+# -*- coding: utf-8 -*-
+import readline
+from Queue import Empty
+
+class ZMQCompleter(object):
+    """Client-side completion machinery.
+
+    How it works: self.complete will be called multiple times, with
+    state=0,1,2,... When state=0 it should compute ALL the completion matches,
+    and then return them for each value of state."""
+    
+    def __init__(self, shell, km):
+        self.shell = shell
+        self.km =  km
+        self.matches = []
+        
+    def complete_request(self,text):
+        line = readline.get_line_buffer()
+        cursor_pos = readline.get_endidx()
+        
+        # send completion request to kernel
+        # Give the kernel up to 0.5s to respond
+        msg_id = self.km.shell_channel.complete(text=text, line=line,
+                                                        cursor_pos=cursor_pos)
+        
+        msg = self.km.shell_channel.get_msg(timeout=0.5)
+        if msg['parent_header']['msg_id'] == msg_id:
+            return msg["content"]["matches"]
+        return []
+    
+    def rlcomplete(self, text, state):
+        if state == 0:
+            try:
+                self.matches = self.complete_request(text)
+            except Empty:
+                print('WARNING: Kernel timeout on tab completion.')
+        
+        try:
+            return self.matches[state]
+        except IndexError:
+            return None
+    
+    def complete(self, text, line, cursor_pos=None):
+        return self.rlcomplete(text, 0)

IPython/frontend/terminal/console/interactiveshell.py

@@ -0,0 +1,342 b''
+# -*- coding: utf-8 -*-
+"""Frontend of ipython working with python-zmq
+
+Ipython's frontend, is a ipython interface that send request to kernel and proccess the kernel's outputs.
+
+For more details, see the ipython-zmq design
+"""
+#-----------------------------------------------------------------------------
+# Copyright (C) 2011 The IPython Development Team
+#
+# Distributed under the terms of the BSD License. The full license is in
+# the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+import bdb
+import signal
+import sys
+import time
+
+from Queue import Empty
+
+from IPython.core.alias import AliasManager, AliasError
+from IPython.core import page
+from IPython.utils.warn import warn, error, fatal
+from IPython.utils import io
+
+from IPython.frontend.terminal.interactiveshell import TerminalInteractiveShell
+from IPython.frontend.terminal.console.completer import ZMQCompleter
+
+
+class ZMQTerminalInteractiveShell(TerminalInteractiveShell):
+    """A subclass of TerminalInteractiveShell that uses the 0MQ kernel"""
+    _executing = False
+    
+    def __init__(self, *args, **kwargs):
+        self.km = kwargs.pop('kernel_manager')
+        self.session_id = self.km.session.session
+        super(ZMQTerminalInteractiveShell, self).__init__(*args, **kwargs)
+        
+    def init_completer(self):
+        """Initialize the completion machinery.
+
+        This creates completion machinery that can be used by client code,
+        either interactively in-process (typically triggered by the readline
+        library), programatically (such as in test suites) or out-of-prcess
+        (typically over the network by remote frontends).
+        """
+        from IPython.core.completerlib import (module_completer,
+                                               magic_run_completer, cd_completer)
+        
+        self.Completer = ZMQCompleter(self, self.km)
+        
+
+        self.set_hook('complete_command', module_completer, str_key = 'import')
+        self.set_hook('complete_command', module_completer, str_key = 'from')
+        self.set_hook('complete_command', magic_run_completer, str_key = '%run')
+        self.set_hook('complete_command', cd_completer, str_key = '%cd')
+
+        # Only configure readline if we truly are using readline.  IPython can
+        # do tab-completion over the network, in GUIs, etc, where readline
+        # itself may be absent
+        if self.has_readline:
+            self.set_readline_completer()
+    
+    def run_cell(self, cell, store_history=True):
+        """Run a complete IPython cell.
+        
+        Parameters
+        ----------
+        cell : str
+          The code (including IPython code such as %magic functions) to run.
+        store_history : bool
+          If True, the raw and translated cell will be stored in IPython's
+          history. For user code calling back into IPython's machinery, this
+          should be set to False.
+        """
+        if (not cell) or cell.isspace():
+            return
+
+        if cell.strip() == 'exit':
+            # explicitly handle 'exit' command
+            return self.ask_exit()
+
+        self._executing = True
+        # flush stale replies, which could have been ignored, due to missed heartbeats
+        while self.km.shell_channel.msg_ready():
+            self.km.shell_channel.get_msg()
+        # shell_channel.execute takes 'hidden', which is the inverse of store_hist
+        msg_id = self.km.shell_channel.execute(cell, not store_history)
+        while not self.km.shell_channel.msg_ready() and self.km.is_alive:
+            try:
+                self.handle_stdin_request(timeout=0.05)
+            except Empty:
+                # display intermediate print statements, etc.
+                self.handle_iopub()
+                pass
+        if self.km.shell_channel.msg_ready():
+            self.handle_execute_reply(msg_id)
+        self._executing = False
+
+    #-----------------
+    # message handlers
+    #-----------------
+
+    def handle_execute_reply(self, msg_id):
+        msg = self.km.shell_channel.get_msg()
+        if msg["parent_header"].get("msg_id", None) == msg_id:
+            
+            self.handle_iopub()
+            
+            content = msg["content"]
+            status = content['status']
+            
+            if status == 'aborted':
+                self.write('Aborted\n')
+                return
+            elif status == 'ok':
+                # print execution payloads as well:
+                for item in content["payload"]:
+                    text = item.get('text', None)
+                    if text:
+                        page.page(text)
+               
+            elif status == 'error':
+                for frame in content["traceback"]:
+                    print(frame, file=io.stderr)
+            
+            self.execution_count = int(content["execution_count"] + 1)
+
+
+    def handle_iopub(self):
+        """ Method to procces subscribe channel's messages
+
+           This method reads a message and processes the content in different
+           outputs like stdout, stderr, pyout and status
+
+           Arguments:
+           sub_msg:  message receive from kernel in the sub socket channel
+                     capture by kernel manager.
+        """
+        while self.km.sub_channel.msg_ready():
+            sub_msg = self.km.sub_channel.get_msg()
+            msg_type = sub_msg['header']['msg_type']
+            parent = sub_msg["parent_header"]
+            if (not parent) or self.session_id == parent['session']:
+                if msg_type == 'status' :
+                    if sub_msg["content"]["execution_state"] == "busy" :
+                        pass
+
+                elif msg_type == 'stream' :
+                    if sub_msg["content"]["name"] == "stdout":
+                        print(sub_msg["content"]["data"], file=io.stdout, end="")
+                        io.stdout.flush()
+                    elif sub_msg["content"]["name"] == "stderr" :
+                        print(sub_msg["content"]["data"], file=io.stderr, end="")
+                        io.stderr.flush()
+
+                elif msg_type == 'pyout':
+                    self.execution_count = int(sub_msg["content"]["execution_count"])
+                    format_dict = sub_msg["content"]["data"]
+                    # taken from DisplayHook.__call__:
+                    hook = self.displayhook
+                    hook.start_displayhook()
+                    hook.write_output_prompt()
+                    hook.write_format_data(format_dict)
+                    hook.log_output(format_dict)
+                    hook.finish_displayhook()
+
+    def handle_stdin_request(self, timeout=0.1):
+        """ Method to capture raw_input
+        """
+        msg_rep = self.km.stdin_channel.get_msg(timeout=timeout)
+        # in case any iopub came while we were waiting:
+        self.handle_iopub()
+        if self.session_id == msg_rep["parent_header"].get("session"):
+            # wrap SIGINT handler
+            real_handler = signal.getsignal(signal.SIGINT)
+            def double_int(sig,frame):
+                # call real handler (forwards sigint to kernel),
+                # then raise local interrupt, stopping local raw_input
+                real_handler(sig,frame)
+                raise KeyboardInterrupt
+            signal.signal(signal.SIGINT, double_int)
+            
+            try:
+                raw_data = raw_input(msg_rep["content"]["prompt"])
+            except EOFError:
+                # turn EOFError into EOF character
+                raw_data = '\x04'
+            except KeyboardInterrupt:
+                sys.stdout.write('\n')
+                return
+            finally:
+                # restore SIGINT handler
+                signal.signal(signal.SIGINT, real_handler)
+            
+            # only send stdin reply if there *was not* another request
+            # or execution finished while we were reading.
+            if not (self.km.stdin_channel.msg_ready() or self.km.shell_channel.msg_ready()):
+                self.km.stdin_channel.input(raw_data)
+
+    def mainloop(self, display_banner=False):
+        while True:
+            try:
+                self.interact(display_banner=display_banner)
+                #self.interact_with_readline()                
+                # XXX for testing of a readline-decoupled repl loop, call
+                # interact_with_readline above
+                break
+            except KeyboardInterrupt:
+                # this should not be necessary, but KeyboardInterrupt
+                # handling seems rather unpredictable...
+                self.write("\nKeyboardInterrupt in interact()\n")
+    
+    def wait_for_kernel(self, timeout=None):
+        """method to wait for a kernel to be ready"""
+        tic = time.time()
+        self.km.hb_channel.unpause()
+        while True:
+            self.run_cell('1', False)
+            if self.km.hb_channel.is_beating():
+                # heart failure was not the reason this returned
+                break
+            else:
+                # heart failed
+                if timeout is not None and (time.time() - tic) > timeout:
+                    return False
+        return True
+    
+    def interact(self, display_banner=None):
+        """Closely emulate the interactive Python console."""
+
+        # batch run -> do not interact
+        if self.exit_now:
+            return
+
+        if display_banner is None:
+            display_banner = self.display_banner
+        
+        if isinstance(display_banner, basestring):
+            self.show_banner(display_banner)
+        elif display_banner:
+            self.show_banner()
+
+        more = False
+        
+        # run a non-empty no-op, so that we don't get a prompt until
+        # we know the kernel is ready. This keeps the connection
+        # message above the first prompt.
+        if not self.wait_for_kernel(3):
+            error("Kernel did not respond\n")
+            return
+        
+        if self.has_readline:
+            self.readline_startup_hook(self.pre_readline)
+            hlen_b4_cell = self.readline.get_current_history_length()
+        else:
+            hlen_b4_cell = 0
+        # exit_now is set by a call to %Exit or %Quit, through the
+        # ask_exit callback.
+
+        while not self.exit_now:
+            if not self.km.is_alive:
+                # kernel died, prompt for action or exit
+                action = "restart" if self.km.has_kernel else "wait for restart"
+                ans = self.ask_yes_no("kernel died, %s ([y]/n)?" % action, default='y')
+                if ans:
+                    if self.km.has_kernel:
+                        self.km.restart_kernel(True)
+                    self.wait_for_kernel(3)
+                else:
+                    self.exit_now = True
+                continue
+            try:
+                # protect prompt block from KeyboardInterrupt
+                # when sitting on ctrl-C
+                self.hooks.pre_prompt_hook()
+                if more:
+                    try:
+                        prompt = self.prompt_manager.render('in2')
+                    except Exception:
+                        self.showtraceback()
+                    if self.autoindent:
+                        self.rl_do_indent = True
+                    
+                else:
+                    try:
+                        prompt = self.separate_in + self.prompt_manager.render('in')
+                    except Exception:
+                        self.showtraceback()
+                
+                line = self.raw_input(prompt)
+                if self.exit_now:
+                    # quick exit on sys.std[in|out] close
+                    break
+                if self.autoindent:
+                    self.rl_do_indent = False
+                    
+            except KeyboardInterrupt:
+                #double-guard against keyboardinterrupts during kbdint handling
+                try:
+                    self.write('\nKeyboardInterrupt\n')
+                    source_raw = self.input_splitter.source_raw_reset()[1]
+                    hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
+                    more = False
+                except KeyboardInterrupt:
+                    pass
+            except EOFError:
+                if self.autoindent:
+                    self.rl_do_indent = False
+                    if self.has_readline:
+                        self.readline_startup_hook(None)
+                self.write('\n')
+                self.exit()
+            except bdb.BdbQuit:
+                warn('The Python debugger has exited with a BdbQuit exception.\n'
+                     'Because of how pdb handles the stack, it is impossible\n'
+                     'for IPython to properly format this particular exception.\n'
+                     'IPython will resume normal operation.')
+            except:
+                # exceptions here are VERY RARE, but they can be triggered
+                # asynchronously by signal handlers, for example.
+                self.showtraceback()
+            else:
+                self.input_splitter.push(line)
+                more = self.input_splitter.push_accepts_more()
+                if (self.SyntaxTB.last_syntax_error and
+                    self.autoedit_syntax):
+                    self.edit_syntax_error()
+                if not more:
+                    source_raw = self.input_splitter.source_reset()
+                    hlen_b4_cell = self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
+                    self.run_cell(source_raw)
+                
+
+        # Turn off the exit flag, so the mainloop can be restarted if desired
+        self.exit_now = False

IPython/frontend/terminal/console/tests/test_console.py

@@ -0,0 +1,59 b''
+"""Tests for two-process terminal frontend
+
+Currenlty only has the most simple test possible, starting a console and running
+a single command.
+
+Authors:
+
+* Min RK
+"""
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import time
+
+import nose.tools as nt
+from nose import SkipTest
+
+from IPython.testing import decorators as dec
+from IPython.testing import tools as tt
+from IPython.utils import py3compat
+from IPython.utils.process import find_cmd
+
+#-----------------------------------------------------------------------------
+# Test functions begin
+#-----------------------------------------------------------------------------
+
[e-mail]_win32
+def test_console_starts():
+    """test that `ipython console` starts a terminal"""
+    from IPython.external import pexpect
+    
+    # weird IOErrors prevent this from firing sometimes:
+    ipython_cmd = None
+    for i in range(5):
+        try:
+            ipython_cmd = find_cmd('ipython3' if py3compat.PY3 else 'ipython')
+        except IOError:
+            time.sleep(0.1)
+        else:
+            break
+    if ipython_cmd is None:
+        raise SkipTest("Could not determine ipython command")
+    
+    p = pexpect.spawn(ipython_cmd, args=['console', '--colors=NoColor'])
+    idx = p.expect([r'In \[\d+\]', pexpect.EOF], timeout=4)
+    nt.assert_equals(idx, 0, "expected in prompt")
+    p.sendline('5')
+    idx = p.expect([r'Out\[\d+\]: 5', pexpect.EOF], timeout=1)
+    nt.assert_equals(idx, 0, "expected out prompt")
+    idx = p.expect([r'In \[\d+\]', pexpect.EOF], timeout=1)
+    nt.assert_equals(idx, 0, "expected second in prompt")
+    # send ctrl-D;ctrl-D to exit
+    p.sendeof()
+    p.sendeof()
+    p.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=1)
+    if p.isalive():
+        p.terminate()

IPython/lib/tests/test_kernel.py

@@ -0,0 +1,63 b''
+"""Tests for kernel utility functions
+
+Authors
+-------
+* MinRK
+"""
+#-----------------------------------------------------------------------------
+# Copyright (c) 2011, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+# Stdlib imports
+from unittest import TestCase
+
+# Third-party imports
+import nose.tools as nt
+
+# Our own imports
+from IPython.testing import decorators as dec
+from IPython.lib import kernel
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
[e-mail]
+def test_swallow_argv():
+    tests = [
+        # expected  , argv       , aliases, flags
+        (['-a', '5'], ['-a', '5'], None, None),
+        (['5'], ['-a', '5'], None, ['a']),
+        ([], ['-a', '5'], ['a'], None),
+        ([], ['-a', '5'], ['a'], ['a']),
+        ([], ['--foo'], None, ['foo']),
+        (['--foo'], ['--foo'], ['foobar'], []),
+        ([], ['--foo', '5'], ['foo'], []),
+        ([], ['--foo=5'], ['foo'], []),
+        (['--foo=5'], ['--foo=5'], [], ['foo']),
+        (['5'], ['--foo', '5'], [], ['foo']),
+        (['bar'], ['--foo', '5', 'bar'], ['foo'], ['foo']),
+        (['bar'], ['--foo=5', 'bar'], ['foo'], ['foo']),
+        (['5','bar'], ['--foo', '5', 'bar'], None, ['foo']),
+        (['bar'], ['--foo', '5', 'bar'], ['foo'], None),
+        (['bar'], ['--foo=5', 'bar'], ['foo'], None),
+    ]
+    for expected, argv, aliases, flags in tests:
+        stripped = kernel.swallow_argv(argv, aliases=aliases, flags=flags)
+        message = '\n'.join(['',
+            "argv: %r" % argv,
+            "aliases: %r" % aliases,
+            "flags : %r" % flags,
+            "expected : %r" % expected,
+            "returned : %r" % stripped,
+        ])
+        yield nt.assert_equal(expected, stripped, message)
+

IPython/frontend/html/notebook/notebookapp.py

@@ -55,6 +55,7 b' from .notebookmanager import NotebookManager'
 from IPython.config.application import catch_config_error
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir
+from IPython.lib.kernel import swallow_argv
 from IPython.zmq.session import Session, default_secure
 from IPython.zmq.zmqshell import ZMQInteractiveShell
 from IPython.zmq.ipkernel import (
@@ -283,27 +284,10 b' class NotebookApp(BaseIPythonApplication):'
         if argv is None:
             argv = sys.argv[1:]
 
-        self.kernel_argv = list(argv) # copy
+        # Scrub frontend-specific flags
+        self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
         # Kernel should inherit default config file from frontend
         self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
-        # Scrub frontend-specific flags
-        for a in argv:
-            if a.startswith('-') and a.lstrip('-') in notebook_flags:
-                self.kernel_argv.remove(a)
-        swallow_next = False
-        for a in argv:
-            if swallow_next:
-                self.kernel_argv.remove(a)
-                swallow_next = False
-                continue
-            if a.startswith('-'):
-                split = a.lstrip('-').split('=')
-                alias = split[0]
-                if alias in notebook_aliases:
-                    self.kernel_argv.remove(a)
-                    if len(split) == 1:
-                        # alias passed with arg via space
-                        swallow_next = True
 
     def init_configurables(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.

IPython/frontend/html/notebook/tests/test_kernelsession.py

@@ -22,5 +22,6 b' class TestKernelManager(TestCase):'
         self.assert_('shell_port' in port_dict)
         self.assert_('hb_port' in port_dict)
         km.get_kernel(kid)
+        km.kill_kernel(kid)
 
 

IPython/frontend/qt/console/qtconsoleapp.py

@@ -11,6 +11,7 b' Authors:'
 * Fernando Perez
 * Bussonnier Matthias
 * Thomas Kluyver
+* Paul Ivanov
 
 """
 
@@ -26,7 +27,7 b' import sys'
 import uuid
 
 # System library imports
-from IPython.external.qt import QtGui
+from IPython.external.qt import QtCore, QtGui
 
 # Local imports
 from IPython.config.application import boolean_flag, catch_config_error
@@ -44,14 +45,14 b' from IPython.utils.py3compat import str_to_bytes'
 from IPython.utils.traitlets import (
     Dict, List, Unicode, Integer, CaselessStrEnum, CBool, Any
 )
-from IPython.zmq.ipkernel import (
-    flags as ipkernel_flags,
-    aliases as ipkernel_aliases,
-    IPKernelApp
-)
+from IPython.zmq.ipkernel import IPKernelApp
 from IPython.zmq.session import Session, default_secure
 from IPython.zmq.zmqshell import ZMQInteractiveShell
 
+from IPython.frontend.consoleapp import (
+        IPythonConsoleApp, app_aliases, app_flags, flags, aliases
+    )
+
 #-----------------------------------------------------------------------------
 # Network Constants
 #-----------------------------------------------------------------------------
@@ -71,10 +72,9 b' ipython qtconsole --pylab=inline # start with pylab in inline plotting mode'
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
-flags = dict(ipkernel_flags)
+# start with copy of flags
+flags = dict(flags)
 qt_flags = {
-    'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
-            "Connect to an existing kernel. If no argument specified, guess most recent"),
     'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
             "Use a pure Python kernel instead of an IPython kernel."),
     'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
@@ -85,27 +85,14 b' qt_flags.update(boolean_flag('
     "use a GUI widget for tab completion",
     "use plaintext output for completion"
 ))
-qt_flags.update(boolean_flag(
-    'confirm-exit', 'IPythonQtConsoleApp.confirm_exit',
-    """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
-       to force a direct exit without any confirmation.
-    """,
-    """Don't prompt the user when exiting. This will terminate the kernel
-       if it is owned by the frontend, and leave it alive if it is external.
-    """
-))
+# and app_flags from the Console Mixin
+qt_flags.update(app_flags)
+# add frontend flags to the full set
 flags.update(qt_flags)
 
-aliases = dict(ipkernel_aliases)
-
+# start with copy of front&backend aliases list
+aliases = dict(aliases)
 qt_aliases = dict(
-    hb = 'IPythonQtConsoleApp.hb_port',
-    shell = 'IPythonQtConsoleApp.shell_port',
-    iopub = 'IPythonQtConsoleApp.iopub_port',
-    stdin = 'IPythonQtConsoleApp.stdin_port',
-    ip = 'IPythonQtConsoleApp.ip',
-    existing = 'IPythonQtConsoleApp.existing',
-    f = 'IPythonQtConsoleApp.connection_file',
 
     style = 'IPythonWidget.syntax_style',
     stylesheet = 'IPythonQtConsoleApp.stylesheet',
@@ -113,10 +100,18 b' qt_aliases = dict('
 
     editor = 'IPythonWidget.editor',
     paging = 'ConsoleWidget.paging',
-    ssh = 'IPythonQtConsoleApp.sshserver',
 )
+# and app_aliases from the Console Mixin
+qt_aliases.update(app_aliases)
+# add frontend aliases to the full set
 aliases.update(qt_aliases)
 
+# get flags&aliases into sets, and remove a couple that
+# shouldn't be scrubbed from backend flags:
+qt_aliases = set(qt_aliases.keys())
+qt_aliases.remove('colors')
+qt_flags = set(qt_flags.keys())
+
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
@@ -126,9 +121,8 b' aliases.update(qt_aliases)'
 #-----------------------------------------------------------------------------
 
 
-class IPythonQtConsoleApp(BaseIPythonApplication):
+class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp):
     name = 'ipython-qtconsole'
-    default_config_file_name='ipython_config.py'
 
     description = """
         The IPython QtConsole.
@@ -150,50 +144,13 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
     classes = [IPKernelApp, IPythonWidget, ZMQInteractiveShell, ProfileDir, Session]
     flags = Dict(flags)
     aliases = Dict(aliases)
-
-    kernel_argv = List(Unicode)
-
-    # create requested profiles by default, if they don't exist:
-    auto_create = CBool(True)
-    # connection info:
-    ip = Unicode(LOCALHOST, config=True,
-        help="""Set the kernel\'s IP address [default localhost].
-        If the IP address is something other than localhost, then
-        Consoles on other machines will be able to connect
-        to the Kernel, so be careful!"""
-    )
-    
-    sshserver = Unicode('', config=True,
-        help="""The SSH server to use to connect to the kernel.""")
-    sshkey = Unicode('', config=True,
-        help="""Path to the ssh key to use for logging in to the ssh server.""")
-    
-    hb_port = Integer(0, config=True,
-        help="set the heartbeat port [default: random]")
-    shell_port = Integer(0, config=True,
-        help="set the shell (XREP) port [default: random]")
-    iopub_port = Integer(0, config=True,
-        help="set the iopub (PUB) port [default: random]")
-    stdin_port = Integer(0, config=True,
-        help="set the stdin (XREQ) port [default: random]")
-    connection_file = Unicode('', config=True,
-        help="""JSON file in which to store connection info [default: kernel-<pid>.json]
-
-        This file will contain the IP, ports, and authentication key needed to connect
-        clients to this kernel. By default, this file will be created in the security-dir
-        of the current profile, but can be specified by absolute path.
-        """)
-    def _connection_file_default(self):
-        return 'kernel-%i.json' % os.getpid()
-
-    existing = Unicode('', config=True,
-        help="""Connect to an already running kernel""")
+    frontend_flags = Any(qt_flags)
+    frontend_aliases = Any(qt_aliases)
+    kernel_manager_class = QtKernelManager
 
     stylesheet = Unicode('', config=True,
         help="path to a custom CSS stylesheet")
 
-    pure = CBool(False, config=True,
-        help="Use a pure Python kernel instead of an IPython kernel.")
     plain = CBool(False, config=True,
         help="Use a plaintext widget instead of rich text (plain can't print/save).")
 
@@ -209,178 +166,13 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
 
     _plain_changed = _pure_changed
 
-    confirm_exit = CBool(True, config=True,
-        help="""
-        Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
-        to force a direct exit without any confirmation.""",
-    )
-    
     # the factory for creating a widget
     widget_factory = Any(RichIPythonWidget)
 
     def parse_command_line(self, argv=None):
         super(IPythonQtConsoleApp, self).parse_command_line(argv)
-        if argv is None:
-            argv = sys.argv[1:]
-        self.kernel_argv = list(argv) # copy
-        # kernel should inherit default config file from frontend
-        self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
-        # Scrub frontend-specific flags
-        swallow_next = False
-        was_flag = False
-        # copy again, in case some aliases have the same name as a flag
-        # argv = list(self.kernel_argv)
-        for a in argv:
-            if swallow_next:
-                swallow_next = False
-                # last arg was an alias, remove the next one
-                # *unless* the last alias has a no-arg flag version, in which
-                # case, don't swallow the next arg if it's also a flag:
-                if not (was_flag and a.startswith('-')):
-                    self.kernel_argv.remove(a)
-                    continue
-            if a.startswith('-'):
-                split = a.lstrip('-').split('=')
-                alias = split[0]
-                if alias in qt_aliases:
-                    self.kernel_argv.remove(a)
-                    if len(split) == 1:
-                        # alias passed with arg via space
-                        swallow_next = True
-                        # could have been a flag that matches an alias, e.g. `existing`
-                        # in which case, we might not swallow the next arg
-                        was_flag = alias in qt_flags
-                elif alias in qt_flags:
-                    # strip flag, but don't swallow next, as flags don't take args
-                    self.kernel_argv.remove(a)
-    
-    def init_connection_file(self):
-        """find the connection file, and load the info if found.
-        
-        The current working directory and the current profile's security
-        directory will be searched for the file if it is not given by
-        absolute path.
-        
-        When attempting to connect to an existing kernel and the `--existing`
-        argument does not match an existing file, it will be interpreted as a
-        fileglob, and the matching file in the current profile's security dir
-        with the latest access time will be used.
-        """
-        if self.existing:
-            try:
-                cf = find_connection_file(self.existing)
-            except Exception:
-                self.log.critical("Could not find existing kernel connection file %s", self.existing)
-                self.exit(1)
-            self.log.info("Connecting to existing kernel: %s" % cf)
-            self.connection_file = cf
-        # should load_connection_file only be used for existing?
-        # as it is now, this allows reusing ports if an existing
-        # file is requested
-        try:
-            self.load_connection_file()
-        except Exception:
-            self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
-            self.exit(1)
-    
-    def load_connection_file(self):
-        """load ip/port/hmac config from JSON connection file"""
-        # this is identical to KernelApp.load_connection_file
-        # perhaps it can be centralized somewhere?
-        try:
-            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
-        except IOError:
-            self.log.debug("Connection File not found: %s", self.connection_file)
-            return
-        self.log.debug(u"Loading connection file %s", fname)
-        with open(fname) as f:
-            s = f.read()
-        cfg = json.loads(s)
-        if self.ip == LOCALHOST and 'ip' in cfg:
-            # not overridden by config or cl_args
-            self.ip = cfg['ip']
-        for channel in ('hb', 'shell', 'iopub', 'stdin'):
-            name = channel + '_port'
-            if getattr(self, name) == 0 and name in cfg:
-                # not overridden by config or cl_args
-                setattr(self, name, cfg[name])
-        if 'key' in cfg:
-            self.config.Session.key = str_to_bytes(cfg['key'])
-    
-    def init_ssh(self):
-        """set up ssh tunnels, if needed."""
-        if not self.sshserver and not self.sshkey:
-            return
+        self.build_kernel_argv(argv)
         
-        if self.sshkey and not self.sshserver:
-            # specifying just the key implies that we are connecting directly
-            self.sshserver = self.ip
-            self.ip = LOCALHOST
-        
-        # build connection dict for tunnels:
-        info = dict(ip=self.ip,
-                    shell_port=self.shell_port,
-                    iopub_port=self.iopub_port,
-                    stdin_port=self.stdin_port,
-                    hb_port=self.hb_port
-        )
-        
-        self.log.info("Forwarding connections to %s via %s"%(self.ip, self.sshserver))
-        
-        # tunnels return a new set of ports, which will be on localhost:
-        self.ip = LOCALHOST
-        try:
-            newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
-        except:
-            # even catch KeyboardInterrupt
-            self.log.error("Could not setup tunnels", exc_info=True)
-            self.exit(1)
-        
-        self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
-        
-        cf = self.connection_file
-        base,ext = os.path.splitext(cf)
-        base = os.path.basename(base)
-        self.connection_file = os.path.basename(base)+'-ssh'+ext
-        self.log.critical("To connect another client via this tunnel, use:")
-        self.log.critical("--existing %s" % self.connection_file)
-    
-    def _new_connection_file(self):
-        return os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % uuid.uuid4())
-
-    def init_kernel_manager(self):
-        # Don't let Qt or ZMQ swallow KeyboardInterupts.
-        signal.signal(signal.SIGINT, signal.SIG_DFL)
-        sec = self.profile_dir.security_dir
-        try:
-            cf = filefind(self.connection_file, ['.', sec])
-        except IOError:
-            # file might not exist
-            if self.connection_file == os.path.basename(self.connection_file):
-                # just shortname, put it in security dir
-                cf = os.path.join(sec, self.connection_file)
-            else:
-                cf = self.connection_file
-
-        # Create a KernelManager and start a kernel.
-        self.kernel_manager = QtKernelManager(
-                                ip=self.ip,
-                                shell_port=self.shell_port,
-                                iopub_port=self.iopub_port,
-                                stdin_port=self.stdin_port,
-                                hb_port=self.hb_port,
-                                connection_file=cf,
-                                config=self.config,
-        )
-        # start the kernel
-        if not self.existing:
-            kwargs = dict(ipython=not self.pure)
-            kwargs['extra_arguments'] = self.kernel_argv
-            self.kernel_manager.start_kernel(**kwargs)
-        elif self.sshserver:
-            # ssh, write new connection file
-            self.kernel_manager.write_connection_file()
-        self.kernel_manager.start_channels()
 
     def new_frontend_master(self):
         """ Create and return new frontend attached to new kernel, launched on localhost.
@@ -517,15 +309,26 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
             else:
                 raise IOError("Stylesheet %r not found."%self.stylesheet)
 
+    def init_signal(self):
+        """allow clean shutdown on sigint"""
+        signal.signal(signal.SIGINT, lambda sig, frame: self.exit(-2))
+        # need a timer, so that QApplication doesn't block until a real
+        # Qt event fires (can require mouse movement)
+        # timer trick from http://stackoverflow.com/q/4938723/938949
+        timer = QtCore.QTimer()
+         # Let the interpreter run each 200 ms:
+        timer.timeout.connect(lambda: None)
+        timer.start(200)
+        # hold onto ref, so the timer doesn't get cleaned up
+        self._sigint_timer = timer
+
     @catch_config_error
     def initialize(self, argv=None):
         super(IPythonQtConsoleApp, self).initialize(argv)
-        self.init_connection_file()
-        default_secure(self.config)
-        self.init_ssh()
-        self.init_kernel_manager()
+        IPythonConsoleApp.initialize(self,argv)
         self.init_qt_elements()
         self.init_colors()
+        self.init_signal()
 
     def start(self):
 

IPython/frontend/terminal/ipapp.py

@@ -69,6 +69,9 b' ipython --profile=foo # start with profile foo'
 ipython qtconsole          # start the qtconsole GUI application
 ipython qtconsole -h       # show the help string for the qtconsole subcmd
 
+ipython console            # start the terminal-based console application
+ipython console -h         # show the help string for the console subcmd
+
 ipython profile create foo # create profile foo w/ default config files
 ipython profile -h         # show the help string for the profile subcmd
 """
@@ -112,7 +115,8 b' class IPAppCrashHandler(CrashHandler):'
 #-----------------------------------------------------------------------------
 flags = dict(base_flags)
 flags.update(shell_flags)
-addflag = lambda *args: flags.update(boolean_flag(*args))
+frontend_flags = {}
+addflag = lambda *args: frontend_flags.update(boolean_flag(*args))
 addflag('autoedit-syntax', 'TerminalInteractiveShell.autoedit_syntax',
         'Turn on auto editing of files with syntax errors.',
         'Turn off auto editing of files with syntax errors.'
@@ -143,7 +147,7 b" classic_config.InteractiveShell.separate_out2 = ''"
 classic_config.InteractiveShell.colors = 'NoColor'
 classic_config.InteractiveShell.xmode = 'Plain'
 
-flags['classic']=(
+frontend_flags['classic']=(
     classic_config,
     "Gives IPython a similar feel to the classic Python prompt."
 )
@@ -153,21 +157,22 b" flags['classic']=("
 #     help="Start logging to the default log file (./ipython_log.py).")
 #
 # # quick is harder to implement
-flags['quick']=(
+frontend_flags['quick']=(
     {'TerminalIPythonApp' : {'quick' : True}},
     "Enable quick startup with no config files."
 )
 
-flags['i'] = (
+frontend_flags['i'] = (
     {'TerminalIPythonApp' : {'force_interact' : True}},
     """If running code from the command line, become interactive afterwards.
     Note: can also be given simply as '-i.'"""
 )
-flags['pylab'] = (
+frontend_flags['pylab'] = (
     {'TerminalIPythonApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
+flags.update(frontend_flags)
 
 aliases = dict(base_aliases)
 aliases.update(shell_aliases)
@@ -209,7 +214,7 b' class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):'
             """Launch the IPython Qt Console."""
         ),
         notebook=('IPython.frontend.html.notebook.notebookapp.NotebookApp',
-            """Launch the IPython HTML Notebook Server"""
+            """Launch the IPython HTML Notebook Server."""
         ),
         profile = ("IPython.core.profileapp.ProfileApp",
             "Create and manage IPython profiles."
@@ -217,6 +222,9 b' class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):'
         kernel = ("IPython.zmq.ipkernel.IPKernelApp",
             "Start a kernel without an attached frontend."
         ),
+        console=('IPython.frontend.terminal.console.app.ZMQTerminalIPythonApp',
+            """Launch the IPython terminal-based Console."""
+        ),
     ))
 
     # *do* autocreate requested profile, but don't create the config file.

IPython/lib/kernel.py

@@ -253,3 +253,63 b' def tunnel_to_kernel(connection_info, sshserver, sshkey=None):'
     return tuple(lports)
     
 
+def swallow_argv(argv, aliases=None, flags=None):
+    """strip frontend-specific aliases and flags from an argument list
+    
+    For use primarily in frontend apps that want to pass a subset of command-line
+    arguments through to a subprocess, where frontend-specific flags and aliases
+    should be removed from the list.
+    
+    Parameters
+    ----------
+    
+    argv : list(str)
+        The starting argv, to be filtered
+    aliases : container of aliases (dict, list, set, etc.)
+        The frontend-specific aliases to be removed
+    flags : container of flags (dict, list, set, etc.)
+        The frontend-specific flags to be removed
+    
+    Returns
+    -------
+    
+    argv : list(str)
+        The argv list, excluding flags and aliases that have been stripped
+    """
+    
+    if aliases is None:
+        aliases = set()
+    if flags is None:
+        flags = set()
+    
+    stripped = list(argv) # copy
+    
+    swallow_next = False
+    was_flag = False
+    for a in argv:
+        if swallow_next:
+            swallow_next = False
+            # last arg was an alias, remove the next one
+            # *unless* the last alias has a no-arg flag version, in which
+            # case, don't swallow the next arg if it's also a flag:
+            if not (was_flag and a.startswith('-')):
+                stripped.remove(a)
+                continue
+        if a.startswith('-'):
+            split = a.lstrip('-').split('=')
+            alias = split[0]
+            if alias in aliases:
+                stripped.remove(a)
+                if len(split) == 1:
+                    # alias passed with arg via space
+                    swallow_next = True
+                    # could have been a flag that matches an alias, e.g. `existing`
+                    # in which case, we might not swallow the next arg
+                    was_flag = alias in flags
+            elif alias in flags and len(split) == 1:
+                # strip flag, but don't swallow next, as flags don't take args
+                stripped.remove(a)
+    
+    # return shortened list
+    return stripped
+

IPython/testing/_paramtestpy2.py

@@ -12,6 +12,7 b''
 # Imports
 #-----------------------------------------------------------------------------
 
+import sys
 import unittest
 from compiler.consts import CO_GENERATOR
 
@@ -45,7 +46,7 b' class ParametricTestCase(unittest.TestCase):'
                 except KeyboardInterrupt:
                     raise
                 except:
-                    result.addError(self, self._exc_info())
+                    result.addError(self, sys.exc_info())
                     return
                 # Test execution
                 ok = False
@@ -56,18 +57,18 b' class ParametricTestCase(unittest.TestCase):'
                     # We stop the loop
                     break
                 except self.failureException:
-                    result.addFailure(self, self._exc_info())
+                    result.addFailure(self, sys.exc_info())
                 except KeyboardInterrupt:
                     raise
                 except:
-                    result.addError(self, self._exc_info())
+                    result.addError(self, sys.exc_info())
                 # TearDown
                 try:
                     self.tearDown()
                 except KeyboardInterrupt:
                     raise
                 except:
-                    result.addError(self, self._exc_info())
+                    result.addError(self, sys.exc_info())
                     ok = False
                 if ok: result.addSuccess(self)
 

IPython/zmq/blockingkernelmanager.py

@@ -16,6 +16,7 b' from __future__ import print_function'
 
 # Stdlib
 from Queue import Queue, Empty
+from threading import Event
 
 # Our own
 from IPython.utils import io
@@ -104,18 +105,42 b' class BlockingShellSocketChannel(ShellSocketChannel):'
 
 class BlockingStdInSocketChannel(StdInSocketChannel):
     
+    def __init__(self, context, session, address=None):
+        super(BlockingStdInSocketChannel, self).__init__(context, session, address)
+        self._in_queue = Queue()
+        
     def call_handlers(self, msg):
         #io.rprint('[[Rep]]', msg) # dbg
-        pass
+        self._in_queue.put(msg)
+        
+    def get_msg(self, block=True, timeout=None):
+        "Gets a message if there is one that is ready."
+        return self._in_queue.get(block, timeout)
+        
+    def get_msgs(self):
+        """Get all messages that are currently ready."""
+        msgs = []
+        while True:
+            try:
+                msgs.append(self.get_msg(block=False))
+            except Empty:
+                break
+        return msgs
+    
+    def msg_ready(self):
+        "Is there a message that has been received?"
+        return not self._in_queue.empty()
 
 
 class BlockingHBSocketChannel(HBSocketChannel):
     
-    # This kernel needs rapid monitoring capabilities
-    time_to_dead = 0.2
+    # This kernel needs quicker monitoring, shorten to 1 sec.
+    # less than 0.5s is unreliable, and will get occasional
+    # false reports of missed beats.
+    time_to_dead = 1.
 
     def call_handlers(self, since_last_heartbeat):
-        #io.rprint('[[Heart]]', since_last_heartbeat) # dbg
+        """pause beating on missed heartbeat"""
         pass
 
 

IPython/zmq/ipkernel.py

@@ -22,7 +22,9 b' import sys'
 import time
 import traceback
 import logging
-
+from signal import (
+        signal, default_int_handler, SIGINT, SIG_IGN
+)
 # System library imports.
 import zmq
 
@@ -168,6 +170,9 b' class Kernel(Configurable):'
     def start(self):
         """ Start the kernel main loop.
         """
+        # a KeyboardInterrupt (SIGINT) can occur on any python statement, so
+        # let's ignore (SIG_IGN) them until we're in a place to handle them properly
+        signal(SIGINT,SIG_IGN)
         poller = zmq.Poller()
         poller.register(self.shell_socket, zmq.POLLIN)
         # loop while self.eventloop has not been overridden
@@ -182,12 +187,20 b' class Kernel(Configurable):'
                 # due to pyzmq Issue #130
                 try:
                     poller.poll(10*1000*self._poll_interval)
+                    # restore raising of KeyboardInterrupt
+                    signal(SIGINT, default_int_handler)
                     self.do_one_iteration()
                 except:
                     raise
+                finally:
+                    # prevent raising of KeyboardInterrupt
+                    signal(SIGINT,SIG_IGN)
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 io.raw_print("KeyboardInterrupt caught in kernel")
+        # stop ignoring sigint, now that we are out of our own loop,
+        # we don't want to prevent future code from handling it
+        signal(SIGINT, default_int_handler)
         if self.eventloop is not None:
             try:
                 self.eventloop(self)
@@ -456,6 +469,9 b' class Kernel(Configurable):'
             self.log.error("Got bad raw_input reply: ")
             self.log.error(str(Message(parent)))
             value = ''
+        if value == '\x04':
+            # EOF
+            raise EOFError
         return value
 
     def _complete(self, msg):

IPython/zmq/kernelmanager.py

@@ -471,83 +471,89 b' class HBSocketChannel(ZMQSocketChannel):'
     poller = None
     _running = None
     _pause = None
+    _beating = None
 
     def __init__(self, context, session, address):
         super(HBSocketChannel, self).__init__(context, session, address)
         self._running = False
         self._pause = True
+        self.poller = zmq.Poller()
 
     def _create_socket(self):
+        if self.socket is not None:
+            # close previous socket, before opening a new one
+            self.poller.unregister(self.socket)
+            self.socket.close()
         self.socket = self.context.socket(zmq.REQ)
-        self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
+        self.socket.setsockopt(zmq.LINGER, 0)
         self.socket.connect('tcp://%s:%i' % self.address)
-        self.poller = zmq.Poller()
+        
         self.poller.register(self.socket, zmq.POLLIN)
 
+    def _poll(self, start_time):
+        """poll for heartbeat replies until we reach self.time_to_dead
+        
+        Ignores interrupts, and returns the result of poll(), which
+        will be an empty list if no messages arrived before the timeout,
+        or the event tuple if there is a message to receive.
+        """
+        
+        until_dead = self.time_to_dead - (time.time() - start_time)
+        # ensure poll at least once
+        until_dead = max(until_dead, 1e-3)
+        events = []
+        while True:
+            try:
+                events = self.poller.poll(1000 * until_dead)
+            except zmq.ZMQError as e:
+                if e.errno == errno.EINTR:
+                    # ignore interrupts during heartbeat
+                    # this may never actually happen
+                    until_dead = self.time_to_dead - (time.time() - start_time)
+                    until_dead = max(until_dead, 1e-3)
+                    pass
+                else:
+                    raise
+            else:
+                break
+        return events
+
     def run(self):
         """The thread's main activity.  Call start() instead."""
         self._create_socket()
         self._running = True
+        self._beating = True
+        
         while self._running:
             if self._pause:
+                # just sleep, and skip the rest of the loop
                 time.sleep(self.time_to_dead)
-            else:
+                continue
+            
                 since_last_heartbeat = 0.0
-                request_time = time.time()
-                try:
                     #io.rprint('Ping from HB channel') # dbg
+            # no need to catch EFSM here, because the previous event was
+            # either a recv or connect, which cannot be followed by EFSM
                     self.socket.send(b'ping')
-                except zmq.ZMQError, e:
-                    #io.rprint('*** HB Error:', e) # dbg
-                    if e.errno == zmq.EFSM:
-                        #io.rprint('sleep...', self.time_to_dead) # dbg
-                        time.sleep(self.time_to_dead)
-                        self._create_socket()
-                    else:
-                        raise
-                else:
-                    while True:
-                        try:
-                            self.socket.recv(zmq.NOBLOCK)
-                        except zmq.ZMQError, e:
-                            #io.rprint('*** HB Error 2:', e) # dbg
-                            if e.errno == zmq.EAGAIN:
-                                before_poll = time.time()
-                                until_dead = self.time_to_dead - (before_poll -
-                                                                  request_time)
-
-                                # When the return value of poll() is an empty
-                                # list, that is when things have gone wrong
-                                # (zeromq bug). As long as it is not an empty
-                                # list, poll is working correctly even if it
-                                # returns quickly. Note: poll timeout is in
-                                # milliseconds.
-                                if until_dead > 0.0:
-                                    while True:
-                                        try:
-                                            self.poller.poll(1000 * until_dead)
-                                        except zmq.ZMQError as e:
-                                            if e.errno == errno.EINTR:
+            request_time = time.time()
+            ready = self._poll(request_time)
+            if ready:
+                self._beating = True
+                # the poll above guarantees we have something to recv
+                self.socket.recv()
+                # sleep the remainder of the cycle
+                remainder = self.time_to_dead - (time.time() - request_time)
+                if remainder > 0:
+                    time.sleep(remainder)
                                                 continue
                                             else:
-                                                raise
-                                        else:
-                                            break
-
+                # nothing was received within the time limit, signal heart failure
+                self._beating = False
                                 since_last_heartbeat = time.time()-request_time
-                                if since_last_heartbeat > self.time_to_dead:
                                     self.call_handlers(since_last_heartbeat)
-                                    break
-                            else:
-                                # FIXME: We should probably log this instead.
-                                raise
-                        else:
-                            until_dead = self.time_to_dead - (time.time() -
-                                                              request_time)
-                            if until_dead > 0.0:
-                                #io.rprint('sleep...', self.time_to_dead) # dbg
-                                time.sleep(until_dead)
-                            break
+                # and close/reopen the socket, because the REQ/REP cycle has been broken
+                self._create_socket()
+                continue
 
     def pause(self):
         """Pause the heartbeat."""
@@ -558,8 +564,8 b' class HBSocketChannel(ZMQSocketChannel):'
         self._pause = False
 
     def is_beating(self):
-        """Is the heartbeat running and not paused."""
-        if self.is_alive() and not self._pause:
+        """Is the heartbeat running and responsive (and not paused)."""
+        if self.is_alive() and not self._pause and self._beating:
             return True
         else:
             return False
@@ -573,7 +579,7 b' class HBSocketChannel(ZMQSocketChannel):'
 
         Subclasses should override this method to handle incoming messages.
         It is important to remember that this method is called in the thread
-        so that some logic must be done to ensure that the application leve
+        so that some logic must be done to ensure that the application level
         handlers are called in the application thread.
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
@@ -639,13 +645,7 b' class KernelManager(HasTraits):'
             self.session = Session(config=self.config)
     
     def __del__(self):
-        if self._connection_file_written:
-            # cleanup connection files on full shutdown of kernel we started
-            self._connection_file_written = False
-            try:
-                os.remove(self.connection_file)
-            except IOError:
-                pass
+        self.cleanup_connection_file()
 
 
     #--------------------------------------------------------------------------
@@ -694,6 +694,19 b' class KernelManager(HasTraits):'
     # Kernel process management methods:
     #--------------------------------------------------------------------------
     
+    def cleanup_connection_file(self):
+        """cleanup connection file *if we wrote it*
+        
+        Will not raise if the connection file was already removed somehow.
+        """
+        if self._connection_file_written:
+            # cleanup connection files on full shutdown of kernel we started
+            self._connection_file_written = False
+            try:
+                os.remove(self.connection_file)
+            except OSError:
+                pass
+    
     def load_connection_file(self):
         """load connection info from JSON dict in self.connection_file"""
         with open(self.connection_file) as f:
@@ -893,16 +906,18 b' class KernelManager(HasTraits):'
     @property
     def is_alive(self):
         """Is the kernel process still running?"""
-        # FIXME: not using a heartbeat means this method is broken for any
-        # remote kernel, it's only capable of handling local kernels.
         if self.has_kernel:
             if self.kernel.poll() is None:
                 return True
             else:
                 return False
+        elif self._hb_channel is not None:
+            # We didn't start the kernel with this KernelManager so we
+            # use the heartbeat.
+            return self._hb_channel.is_beating()
         else:
-            # We didn't start the kernel with this KernelManager so we don't
-            # know if it is running. We should use a heartbeat for this case.
+            # no heartbeat and not local, we can't tell if it's running,
+            # so naively return True
             return True
 
     #--------------------------------------------------------------------------

IPython/zmq/zmqshell.py

@@ -44,12 +44,6 b' from IPython.zmq.displayhook import ZMQShellDisplayHook, _encode_binary'
 from IPython.zmq.session import extract_header
 from session import Session
 
-#-----------------------------------------------------------------------------
-# Globals and side-effects
-#-----------------------------------------------------------------------------
-
-# Install the payload version of page.
-install_payload_page()
 
 #-----------------------------------------------------------------------------
 # Functions and classes
@@ -127,6 +121,9 b' class ZMQInteractiveShell(InteractiveShell):'
         env['PAGER'] = 'cat'
         env['GIT_PAGER'] = 'cat'
 
+        # And install the payload version of page.
+        install_payload_page()
+
     def auto_rewrite_input(self, cmd):
         """Called to show the auto-rewritten input for autocall and friends.