use connection files instead of ports to connect to kernels

this addresses security issues (#688)
and ease-of-multi-frontend

IPython/frontend/qt/console/qtconsoleapp.py

@@ -17,6 +17,7 b' Authors:'
 #-----------------------------------------------------------------------------
 
 # stdlib imports
+import glob
 import os
 import signal
 import sys
@@ -25,6 +26,7 b' from getpass import getpass'
 # System library imports
 from IPython.external.qt import QtGui
 from pygments.styles import get_all_styles
+from zmq.utils import jsonapi as json
 
 # external imports
 from IPython.external.ssh import tunnel
@@ -39,6 +41,7 b' from IPython.frontend.qt.console.rich_ipython_widget import RichIPythonWidget'
 from IPython.frontend.qt.console import styles
 from IPython.frontend.qt.kernelmanager import QtKernelManager
 from IPython.parallel.util import select_random_ports
+from IPython.utils.path import filefind
 from IPython.utils.traitlets import (
     Dict, List, Unicode, Int, CaselessStrEnum, CBool, Any
 )
@@ -182,8 +185,8 b' class MainWindow(QtGui.QMainWindow):'
 
 flags = dict(ipkernel_flags)
 qt_flags = {
-    'existing' : ({'IPythonQtConsoleApp' : {'existing' : True}},
-            "Connect to an existing kernel."),
+    'existing' : ({'IPythonQtConsoleApp' : {'existing' : 'kernel*.json'}},
+            "Connect to an existing kernel. If no argument specified, guess most recent"),
     'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}},
             "Use a pure Python kernel instead of an IPython kernel."),
     'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}},
@@ -213,6 +216,8 b' qt_aliases = dict('
     iopub = 'IPythonQtConsoleApp.iopub_port',
     stdin = 'IPythonQtConsoleApp.stdin_port',
     ip = 'IPythonQtConsoleApp.ip',
+    existing = 'IPythonQtConsoleApp.existing',
+    f = 'IPythonQtConsoleApp.connection_file',
 
     style = 'IPythonWidget.syntax_style',
     stylesheet = 'IPythonQtConsoleApp.stylesheet',
@@ -280,9 +285,18 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
         help="set the iopub (PUB) port [default: random]")
     stdin_port = Int(0, config=True,
         help="set the stdin (XREQ) port [default: random]")
+    connection_file = Unicode('', config=True,
+        help="""JSON file in which to store connection info [default: kernel-<pid>.json]
 
-    existing = CBool(False, config=True,
-        help="Whether to connect to an already running Kernel.")
+        This file will contain the IP, ports, and authentication key needed to connect
+        clients to this kernel. By default, this file will be created in the security-dir
+        of the current profile, but can be specified by absolute path.
+        """)
+    def _connection_file_default(self):
+        return 'kernel-%i.json' % os.getpid()
+
+    existing = Unicode('', config=True,
+        help="""Connect to an already running kernel""")
 
     stylesheet = Unicode('', config=True,
         help="path to a custom CSS stylesheet")
@@ -340,6 +354,58 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
                         # alias passed with arg via space
                         swallow_next = True
     
+    def init_connection_file(self):
+        sec = self.profile_dir.security_dir
+        if self.existing:
+            try:
+                # first, try explicit name
+                cf = filefind(self.existing, ['.', sec])
+            except IOError:
+                # not found by full name
+                if '*' in self.existing:
+                    # given as a glob already
+                    pat = self.existing
+                else:
+                    # accept any substring match
+                    pat = '*%s*'
+                matches = glob.glob( os.path.join(sec, pat) )
+                if not matches:
+                    self.log.critical("Could not find existing kernel connection file %s", self.existing)
+                    self.exit(1)
+                else:
+                    # get most recent match:
+                    cf = sorted(matches, key=lambda f: os.stat(f).st_atime)[-1]
+            self.log.info("Connecting to existing kernel: %s" % cf)
+            self.connection_file = cf
+        # should load_connection_file only be used for existing?
+        # as it is now, this allows reusing ports if an existing
+        # file is requested
+        self.load_connection_file()
+    
+    def load_connection_file(self):
+        """load ip/port/hmac config from JSON connection file"""
+        # this is identical to KernelApp.load_connection_file
+        # perhaps it can be centralized somewhere?
+        try:
+            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+        except IOError:
+            self.log.debug("Connection File not found: %s", self.connection_file)
+            return
+        self.log.debug(u"Loading connection file %s", fname)
+        with open(fname) as f:
+            s = f.read()
+        cfg = json.loads(s)
+        if self.ip == LOCALHOST and 'ip' in cfg:
+            # not overridden by config or cl_args
+            self.ip = cfg['ip']
+        for channel in ('hb', 'shell', 'iopub', 'stdin'):
+            name = channel + '_port'
+            if getattr(self, name) == 0 and name in cfg:
+                # not overridden by config or cl_args
+                setattr(self, name, cfg[name])
+        if 'key' in cfg:
+            self.config.Session.key = cfg['key']
+    
     def init_ssh(self):
         """set up ssh tunnels, if needed."""
         if not self.sshserver and not self.sshkey:
@@ -374,18 +440,30 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
     def init_kernel_manager(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.
         signal.signal(signal.SIGINT, signal.SIG_DFL)
+        sec = self.profile_dir.security_dir
+        try:
+            cf = filefind(self.connection_file, ['.', sec])
+        except IOError:
+            # file might not exist, use 
+            if self.connection_file == os.path.basename(self.connection_file):
+                # just shortname, put it in security dir
+                cf = os.path.join(sec, self.connection_file)
+            else:
+                cf = self.connection_file
 
         # Create a KernelManager and start a kernel.
-        self.kernel_manager = QtKernelManager(ip=self.ip,
+        self.kernel_manager = QtKernelManager(
+                                ip=self.ip,
                                 shell_port=self.shell_port,
                                 sub_port=self.iopub_port,
                                 stdin_port=self.stdin_port,
                                 hb_port=self.hb_port,
+                                connection_file=cf,
                                 config=self.config,
         )
         # start the kernel
         if not self.existing:
-            kwargs = dict(ip=self.ip, ipython=not self.pure)
+            kwargs = dict(ipython=not self.pure)
             kwargs['extra_arguments'] = self.kernel_argv
             self.kernel_manager.start_kernel(**kwargs)
         self.kernel_manager.start_channels()
@@ -469,6 +547,7 b' class IPythonQtConsoleApp(BaseIPythonApplication):'
     def initialize(self, argv=None):
         super(IPythonQtConsoleApp, self).initialize(argv)
         self.init_ssh()
+        self.init_connection_file()
         self.init_kernel_manager()
         self.init_qt_elements()
         self.init_colors()

IPython/zmq/entry_point.py

@@ -8,20 +8,26 b' import os'
 import socket
 from subprocess import Popen, PIPE
 import sys
+import tempfile
 
-# Local imports.
-from parentpoller import ParentPollerWindows
+# System library imports
+from zmq.utils import jsonapi as json
 
+# IPython imports
+from IPython.utils.localinterfaces import LOCALHOST
 
-def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
-                        ip=None, stdin=None, stdout=None, stderr=None,
-                        executable=None, independent=False, extra_arguments=[]):
-    """ Launches a localhost kernel, binding to the specified ports.
+# Local imports.
+from parentpoller import ParentPollerWindows
 
+def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
+                         ip=LOCALHOST, key=''):
+    """Generates a JSON config file, including the selection of random ports.
+    
     Parameters
     ----------
-    code : str,
-        A string of Python code that imports and executes a kernel entry point.
+
+    fname : unicode
+        The path to the file to write
 
     shell_port : int, optional
         The port to use for XREP channel.
@@ -38,27 +44,14 b' def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0'
     ip  : str, optional
         The ip address the kernel will bind to.
 
-    stdin, stdout, stderr : optional (default None)
-        Standards streams, as defined in subprocess.Popen.
+    key : str, optional
+        The Session key used for HMAC authentication.
 
-    executable : str, optional (default sys.executable)
-        The Python executable to use for the kernel process.
-
-    independent : bool, optional (default False)
-        If set, the kernel process is guaranteed to survive if this process
-        dies. If not set, an effort is made to ensure that the kernel is killed
-        when this process dies. Note that in this case it is still good practice
-        to kill kernels manually before exiting.
-
-    extra_arguments = list, optional
-        A list of extra arguments to pass when executing the launch code.
-
-    Returns
-    -------
-    A tuple of form:
-        (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
-    where kernel_process is a Popen object and the ports are integers.
     """
+    # default to temporary connector file
+    if not fname:
+        fname = tempfile.mktemp('.json')
+    
     # Find open ports as necessary.
     ports = []
     ports_needed = int(shell_port <= 0) + int(iopub_port <= 0) + \
@@ -79,15 +72,62 b' def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0'
         stdin_port = ports.pop(0)
     if hb_port <= 0:
         hb_port = ports.pop(0)
+    
+    cfg = dict( shell_port=shell_port,
+                iopub_port=iopub_port,
+                stdin_port=stdin_port,
+                hb_port=hb_port,
+              )
+    cfg['ip'] = ip
+    cfg['key'] = key
+    
+    with open(fname, 'wb') as f:
+        f.write(json.dumps(cfg, indent=2))
+    
+    return fname, cfg
+    
+
+def base_launch_kernel(code, fname, stdin=None, stdout=None, stderr=None,
+                        executable=None, independent=False, extra_arguments=[]):
+    """ Launches a localhost kernel, binding to the specified ports.
+
+    Parameters
+    ----------
+    code : str,
+        A string of Python code that imports and executes a kernel entry point.
 
+    stdin, stdout, stderr : optional (default None)
+        Standards streams, as defined in subprocess.Popen.
+
+    fname : unicode, optional
+        The JSON connector file, containing ip/port/hmac key information.
+
+    key : str, optional
+        The Session key used for HMAC authentication.
+
+    executable : str, optional (default sys.executable)
+        The Python executable to use for the kernel process.
+
+    independent : bool, optional (default False)
+        If set, the kernel process is guaranteed to survive if this process
+        dies. If not set, an effort is made to ensure that the kernel is killed
+        when this process dies. Note that in this case it is still good practice
+        to kill kernels manually before exiting.
+
+    extra_arguments = list, optional
+        A list of extra arguments to pass when executing the launch code.
+
+    Returns
+    -------
+    A tuple of form:
+        (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
+    where kernel_process is a Popen object and the ports are integers.
+    """
+    
     # Build the kernel launch command.
     if executable is None:
         executable = sys.executable
-    arguments = [ executable, '-c', code, '--shell=%i'%shell_port,
-                  '--iopub=%i'%iopub_port, '--stdin=%i'%stdin_port,
-                  '--hb=%i'%hb_port ]
-    if ip is not None:
-        arguments.append('--ip=%s'%ip)
+    arguments = [ executable, '-c', code, '-f', fname ]
     arguments.extend(extra_arguments)
 
     # Popen will fail (sometimes with a deadlock) if stdin, stdout, and stderr
@@ -164,4 +204,4 b' def base_launch_kernel(code, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0'
         if stderr is None:
             proc.stderr.close()
 
-    return proc, shell_port, iopub_port, stdin_port, hb_port
+    return proc

IPython/zmq/kernelapp.py

@@ -21,6 +21,7 b' import sys'
 
 # System library imports.
 import zmq
+from zmq.utils import jsonapi as json
 
 # IPython imports.
 from IPython.core.ultratb import FormattedTB
@@ -29,10 +30,12 b' from IPython.core.application import ('
 )
 from IPython.utils import io
 from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.path import filefind
 from IPython.utils.traitlets import (Any, Instance, Dict, Unicode, Int, Bool,
                                         DottedObjectName)
 from IPython.utils.importstring import import_item
 # local imports
+from IPython.zmq.entry_point import write_connection_file
 from IPython.zmq.heartbeat import Heartbeat
 from IPython.zmq.parentpoller import ParentPollerUnix, ParentPollerWindows
 from IPython.zmq.session import Session
@@ -49,6 +52,7 b' kernel_aliases.update({'
     'shell' : 'KernelApp.shell_port',
     'iopub' : 'KernelApp.iopub_port',
     'stdin' : 'KernelApp.stdin_port',
+    'f' : 'KernelApp.connection_file',
     'parent': 'KernelApp.parent',
 })
 if sys.platform.startswith('win'):
@@ -99,6 +103,13 b' class KernelApp(BaseIPythonApplication):'
     shell_port = Int(0, config=True, help="set the shell (XREP) port [default: random]")
     iopub_port = Int(0, config=True, help="set the iopub (PUB) port [default: random]")
     stdin_port = Int(0, config=True, help="set the stdin (XREQ) port [default: random]")
+    connection_file = Unicode('', config=True, 
+    help="""JSON file in which to store connection info [default: kernel-<pid>.json]
+    
+    This file will contain the IP, ports, and authentication key needed to connect
+    clients to this kernel. By default, this file will be created in the security-dir
+    of the current profile, but can be specified by absolute path.
+    """)
 
     # streams, etc.
     no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
@@ -138,6 +149,44 b' class KernelApp(BaseIPythonApplication):'
             s.bind(iface + ':%i'%port)
         return port
 
+    def load_connection_file(self):
+        """load ip/port/hmac config from JSON connection file"""
+        try:
+            fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
+        except IOError:
+            self.log.debug("Connection file not found: %s", self.connection_file)
+            return
+        self.log.debug(u"Loading connection file %s", fname)
+        with open(fname) as f:
+            s = f.read()
+        cfg = json.loads(s)
+        if self.ip == LOCALHOST and 'ip' in cfg:
+            # not overridden by config or cl_args
+            self.ip = cfg['ip']
+        for channel in ('hb', 'shell', 'iopub', 'stdin'):
+            name = channel + '_port'
+            if getattr(self, name) == 0 and name in cfg:
+                # not overridden by config or cl_args
+                setattr(self, name, cfg[name])
+        if 'key' in cfg:
+            self.config.Session.key = cfg['key']
+    
+    def write_connection_file(self):
+        """write connection info to JSON file"""
+        if os.path.basename(self.connection_file) == self.connection_file:
+            cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
+        else:
+            cf = self.connection_file
+        write_connection_file(cf, ip=self.ip, key=self.session.key,
+        shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
+        iopub_port=self.iopub_port)
+    
+    def init_connection_file(self):
+        if not self.connection_file:
+            self.connection_file = "kernel-%s.json"%os.getpid()
+        
+        self.load_connection_file()
+
     def init_sockets(self):
         # Create a context, a session, and the kernel sockets.
         self.log.info("Starting the kernel at pid: %i", os.getpid())
@@ -161,12 +210,17 b' class KernelApp(BaseIPythonApplication):'
         self.hb_port = self.heartbeat.port
         self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port)
 
-        # Helper to make it easier to connect to an existing kernel, until we have
-        # single-port connection negotiation fully implemented.
+        # Helper to make it easier to connect to an existing kernel.
         # set log-level to critical, to make sure it is output
         self.log.critical("To connect another client to this kernel, use:")
-        self.log.critical("--existing --shell={0} --iopub={1} --stdin={2} --hb={3}".format(
-            self.shell_port, self.iopub_port, self.stdin_port, self.hb_port))
+        if os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
+            # use shortname
+            tail = os.path.basename(self.connection_file)
+            if self.profile != 'default':
+                tail += " --profile %s" % self.profile_name
+        else:
+            tail = self.connection_file
+        self.log.critical("--existing %s", tail)
 
 
         self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
@@ -209,9 +263,12 b' class KernelApp(BaseIPythonApplication):'
     def initialize(self, argv=None):
         super(KernelApp, self).initialize(argv)
         self.init_blackhole()
+        self.init_connection_file()
         self.init_session()
         self.init_poller()
         self.init_sockets()
+        # writing connection file must be *after* init_sockets
+        self.write_connection_file()
         self.init_io()
         self.init_kernel()
 

IPython/zmq/kernelmanager.py

@@ -19,6 +19,7 b' TODO'
 import errno
 from Queue import Queue, Empty
 from subprocess import Popen
+import os
 import signal
 import sys
 from threading import Thread
@@ -32,7 +33,10 b' from zmq.eventloop import ioloop'
 # Local imports.
 from IPython.config.loader import Config
 from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
-from IPython.utils.traitlets import HasTraits, Any, Instance, Type, Unicode, Int
+from IPython.utils.traitlets import (
+    HasTraits, Any, Instance, Type, Unicode, Int, Bool
+)
+from IPython.zmq.entry_point import write_connection_file
 from session import Session
 
 #-----------------------------------------------------------------------------
@@ -702,6 +706,7 b' class KernelManager(HasTraits):'
     kernel = Instance(Popen)
 
     # The addresses for the communication channels.
+    connection_file = Unicode('')
     ip = Unicode(LOCALHOST)
     shell_port = Int(0)
     sub_port = Int(0)
@@ -720,13 +725,22 b' class KernelManager(HasTraits):'
     _sub_channel = Any
     _stdin_channel = Any
     _hb_channel = Any
+    _connection_file_written=Bool(False)
 
     def __init__(self, **kwargs):
         super(KernelManager, self).__init__(**kwargs)
         if self.session is None:
             self.session = Session(config=self.config)
-        # Uncomment this to try closing the context.
-        # atexit.register(self.context.term)
+    
+    def __del__(self):
+        if self._connection_file_written:
+            # cleanup connection files on full shutdown of kernel we started
+            self._connection_file_written = False
+            try:
+                os.remove(self.connection_file)
+            except IOError:
+                pass
+
 
     #--------------------------------------------------------------------------
     # Channel management methods:
@@ -773,7 +787,22 b' class KernelManager(HasTraits):'
     #--------------------------------------------------------------------------
     # Kernel process management methods:
     #--------------------------------------------------------------------------
-
+    
+    def write_connection_file(self):
+        if self._connection_file_written:
+            return
+        self.connection_file,cfg = write_connection_file(self.connection_file,
+            ip=self.ip, key=self.session.key,
+            stdin_port=self.stdin_port, iopub_port=self.sub_port,
+            shell_port=self.shell_port, hb_port=self.hb_port)
+        # write_connection_file also sets default ports:
+        self.shell_port = cfg['shell_port']
+        self.stdin_port = cfg['stdin_port']
+        self.sub_port = cfg['iopub_port']
+        self.hb_port = cfg['hb_port']
+        
+        self._connection_file_written = True
+    
     def start_kernel(self, **kw):
         """Starts a kernel process and configures the manager to use it.
 
@@ -799,6 +828,9 b' class KernelManager(HasTraits):'
                                "configured properly. "
                                "Currently valid addresses are: %s"%LOCAL_IPS
                                )
+        
+        # write connection file / get default ports
+        self.write_connection_file()
 
         self._launch_args = kw.copy()
         launch_kernel = kw.pop('launcher', None)
@@ -807,13 +839,7 b' class KernelManager(HasTraits):'
                 from ipkernel import launch_kernel
             else:
                 from pykernel import launch_kernel
-        self.kernel, shell, sub, stdin, hb = launch_kernel(
-            shell_port=self.shell_port, iopub_port=self.sub_port,
-            stdin_port=self.stdin_port, hb_port=self.hb_port, **kw)
-        self.shell_port = shell
-        self.sub_port = sub
-        self.stdin_port = stdin
-        self.hb_port = hb
+        self.kernel = launch_kernel(fname=self.connection_file, **kw)
 
     def shutdown_kernel(self, restart=False):
         """ Attempts to the stop the kernel process cleanly. If the kernel
@@ -842,6 +868,14 b' class KernelManager(HasTraits):'
             if self.has_kernel:
                 self.kill_kernel()
 
+        if not restart and self._connection_file_written:
+            # cleanup connection files on full shutdown of kernel we started
+            self._connection_file_written = False
+            try:
+                os.remove(self.connection_file)
+            except IOError:
+                pass
+
     def restart_kernel(self, now=False, **kw):
         """Restarts a kernel with the arguments that were used to launch it.