Merge pull request #7832 from minrk/default-secure-session

remove default_secure, make Session secure by default

IPython/consoleapp.py

@@ -1,345 +1,344 b''
 """ A minimal application base mixin for all ZMQ based IPython frontends.
 
 This is not a complete console app, as subprocess will not be able to receive
 input, there is no real readline support, among other limitations. This is a
 refactoring of what used to be the IPython/qt/console/qtconsoleapp.py
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import atexit
 import os
 import signal
 import sys
 import uuid
 
 
 from IPython.config.application import boolean_flag
 from IPython.core.profiledir import ProfileDir
 from IPython.kernel.blocking import BlockingKernelClient
 from IPython.kernel import KernelManager
 from IPython.kernel import tunnel_to_kernel, find_connection_file, swallow_argv
 from IPython.kernel.kernelspec import NoSuchKernel
 from IPython.utils.path import filefind
 from IPython.utils.traitlets import (
     Dict, List, Unicode, CUnicode, CBool, Any
 )
 from IPython.kernel.zmq.kernelapp import (
     kernel_flags,
     kernel_aliases,
     IPKernelApp
 )
 from IPython.kernel.zmq.pylab.config import InlineBackend
-from IPython.kernel.zmq.session import Session, default_secure
+from IPython.kernel.zmq.session import Session
 from IPython.kernel.zmq.zmqshell import ZMQInteractiveShell
 from IPython.kernel.connect import ConnectionFileMixin
 
 from IPython.utils.localinterfaces import localhost
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = dict(kernel_flags)
 
 # the flags that are specific to the frontend
 # these must be scrubbed before being passed to the kernel,
 # or it will raise an error on unrecognized flags
 app_flags = {
     'existing' : ({'IPythonConsoleApp' : {'existing' : 'kernel*.json'}},
             "Connect to an existing kernel. If no argument specified, guess most recent"),
 }
 app_flags.update(boolean_flag(
     'confirm-exit', 'IPythonConsoleApp.confirm_exit',
     """Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
        to force a direct exit without any confirmation.
     """,
     """Don't prompt the user when exiting. This will terminate the kernel
        if it is owned by the frontend, and leave it alive if it is external.
     """
 ))
 flags.update(app_flags)
 
 aliases = dict(kernel_aliases)
 
 # also scrub aliases from the frontend
 app_aliases = dict(
     ip = 'IPythonConsoleApp.ip',
     transport = 'IPythonConsoleApp.transport',
     hb = 'IPythonConsoleApp.hb_port',
     shell = 'IPythonConsoleApp.shell_port',
     iopub = 'IPythonConsoleApp.iopub_port',
     stdin = 'IPythonConsoleApp.stdin_port',
     existing = 'IPythonConsoleApp.existing',
     f = 'IPythonConsoleApp.connection_file',
 
     kernel = 'IPythonConsoleApp.kernel_name',
 
     ssh = 'IPythonConsoleApp.sshserver',
 )
 aliases.update(app_aliases)
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 classes = [KernelManager, ProfileDir, Session]
 
 class IPythonConsoleApp(ConnectionFileMixin):
     name = 'ipython-console-mixin'
 
     description = """
         The IPython Mixin Console.
         
         This class contains the common portions of console client (QtConsole,
         ZMQ-based terminal console, etc).  It is not a full console, in that
         launched terminal subprocesses will not be able to accept input.
         
         The Console using this mixing supports various extra features beyond
         the single-process Terminal IPython shell, such as connecting to
         existing kernel, via:
         
             ipython <appname> --existing
         
         as well as tunnel via SSH
         
     """
 
     classes = classes
     flags = Dict(flags)
     aliases = Dict(aliases)
     kernel_manager_class = KernelManager
     kernel_client_class = BlockingKernelClient
 
     kernel_argv = List(Unicode)
     # frontend flags&aliases to be stripped when building kernel_argv
     frontend_flags = Any(app_flags)
     frontend_aliases = Any(app_aliases)
 
     # create requested profiles by default, if they don't exist:
     auto_create = CBool(True)
     # connection info:
     
     sshserver = Unicode('', config=True,
         help="""The SSH server to use to connect to the kernel.""")
     sshkey = Unicode('', config=True,
         help="""Path to the ssh key to use for logging in to the ssh server.""")
     
     def _connection_file_default(self):
         return 'kernel-%i.json' % os.getpid()
 
     existing = CUnicode('', config=True,
         help="""Connect to an already running kernel""")
 
     kernel_name = Unicode('python', config=True,
         help="""The name of the default kernel to start.""")
 
     confirm_exit = CBool(True, config=True,
         help="""
         Set to display confirmation dialog on exit. You can always use 'exit' or 'quit',
         to force a direct exit without any confirmation.""",
     )
     
     @property
     def help_classes(self):
         """ConsoleApps can configure kernels on the command-line
         
         But this shouldn't be written to a file
         """
         return self.classes + [IPKernelApp] + IPKernelApp.classes
     
     def build_kernel_argv(self, argv=None):
         """build argv to be passed to kernel subprocess"""
         if argv is None:
             argv = sys.argv[1:]
         self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags)
     
     def init_connection_file(self):
         """find the connection file, and load the info if found.
         
         The current working directory and the current profile's security
         directory will be searched for the file if it is not given by
         absolute path.
         
         When attempting to connect to an existing kernel and the `--existing`
         argument does not match an existing file, it will be interpreted as a
         fileglob, and the matching file in the current profile's security dir
         with the latest access time will be used.
         
         After this method is called, self.connection_file contains the *full path*
         to the connection file, never just its name.
         """
         if self.existing:
             try:
                 cf = find_connection_file(self.existing)
             except Exception:
                 self.log.critical("Could not find existing kernel connection file %s", self.existing)
                 self.exit(1)
             self.log.debug("Connecting to existing kernel: %s" % cf)
             self.connection_file = cf
         else:
             # not existing, check if we are going to write the file
             # and ensure that self.connection_file is a full path, not just the shortname
             try:
                 cf = find_connection_file(self.connection_file)
             except Exception:
                 # file might not exist
                 if self.connection_file == os.path.basename(self.connection_file):
                     # just shortname, put it in security dir
                     cf = os.path.join(self.profile_dir.security_dir, self.connection_file)
                 else:
                     cf = self.connection_file
                 self.connection_file = cf
         try:
             self.connection_file = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
         except IOError:
             self.log.debug("Connection File not found: %s", self.connection_file)
             return
         
         # should load_connection_file only be used for existing?
         # as it is now, this allows reusing ports if an existing
         # file is requested
         try:
             self.load_connection_file()
         except Exception:
             self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
             self.exit(1)
     
     def init_ssh(self):
         """set up ssh tunnels, if needed."""
         if not self.existing or (not self.sshserver and not self.sshkey):
             return
         self.load_connection_file()
         
         transport = self.transport
         ip = self.ip
         
         if transport != 'tcp':
             self.log.error("Can only use ssh tunnels with TCP sockets, not %s", transport)
             sys.exit(-1)
         
         if self.sshkey and not self.sshserver:
             # specifying just the key implies that we are connecting directly
             self.sshserver = ip
             ip = localhost()
         
         # build connection dict for tunnels:
         info = dict(ip=ip,
                     shell_port=self.shell_port,
                     iopub_port=self.iopub_port,
                     stdin_port=self.stdin_port,
                     hb_port=self.hb_port
         )
         
         self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver))
         
         # tunnels return a new set of ports, which will be on localhost:
         self.ip = localhost()
         try:
             newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
         except:
             # even catch KeyboardInterrupt
             self.log.error("Could not setup tunnels", exc_info=True)
             self.exit(1)
         
         self.shell_port, self.iopub_port, self.stdin_port, self.hb_port = newports
         
         cf = self.connection_file
         base,ext = os.path.splitext(cf)
         base = os.path.basename(base)
         self.connection_file = os.path.basename(base)+'-ssh'+ext
         self.log.info("To connect another client via this tunnel, use:")
         self.log.info("--existing %s" % self.connection_file)
     
     def _new_connection_file(self):
         cf = ''
         while not cf:
             # we don't need a 128b id to distinguish kernels, use more readable
             # 48b node segment (12 hex chars).  Users running more than 32k simultaneous
             # kernels can subclass.
             ident = str(uuid.uuid4()).split('-')[-1]
             cf = os.path.join(self.profile_dir.security_dir, 'kernel-%s.json' % ident)
             # only keep if it's actually new.  Protect against unlikely collision
             # in 48b random search space
             cf = cf if not os.path.exists(cf) else ''
         return cf
 
     def init_kernel_manager(self):
         # Don't let Qt or ZMQ swallow KeyboardInterupts.
         if self.existing:
             self.kernel_manager = None
             return
         signal.signal(signal.SIGINT, signal.SIG_DFL)
 
         # Create a KernelManager and start a kernel.
         try:
             self.kernel_manager = self.kernel_manager_class(
                                     ip=self.ip,
                                     session=self.session,
                                     transport=self.transport,
                                     shell_port=self.shell_port,
                                     iopub_port=self.iopub_port,
                                     stdin_port=self.stdin_port,
                                     hb_port=self.hb_port,
                                     connection_file=self.connection_file,
                                     kernel_name=self.kernel_name,
                                     parent=self,
                                     ipython_dir=self.ipython_dir,
             )
         except NoSuchKernel:
             self.log.critical("Could not find kernel %s", self.kernel_name)
             self.exit(1)
 
         self.kernel_manager.client_factory = self.kernel_client_class
         # FIXME: remove special treatment of IPython kernels
         kwargs = {}
         if self.kernel_manager.ipython_kernel:
             kwargs['extra_arguments'] = self.kernel_argv
         self.kernel_manager.start_kernel(**kwargs)
         atexit.register(self.kernel_manager.cleanup_ipc_files)
 
         if self.sshserver:
             # ssh, write new connection file
             self.kernel_manager.write_connection_file()
 
         # in case KM defaults / ssh writing changes things:
         km = self.kernel_manager
         self.shell_port=km.shell_port
         self.iopub_port=km.iopub_port
         self.stdin_port=km.stdin_port
         self.hb_port=km.hb_port
         self.connection_file = km.connection_file
 
         atexit.register(self.kernel_manager.cleanup_connection_file)
 
     def init_kernel_client(self):
         if self.kernel_manager is not None:
             self.kernel_client = self.kernel_manager.client()
         else:
             self.kernel_client = self.kernel_client_class(
                                 session=self.session,
                                 ip=self.ip,
                                 transport=self.transport,
                                 shell_port=self.shell_port,
                                 iopub_port=self.iopub_port,
                                 stdin_port=self.stdin_port,
                                 hb_port=self.hb_port,
                                 connection_file=self.connection_file,
                                 parent=self,
             )
 
         self.kernel_client.start_channels()
 
 
 
     def initialize(self, argv=None):
         """
         Classes which mix this class in should call:
                IPythonConsoleApp.initialize(self,argv)
         """
         self.init_connection_file()
-        default_secure(self.config)
         self.init_ssh()
         self.init_kernel_manager()
         self.init_kernel_client()
 

IPython/html/notebookapp.py

@@ -1,1118 +1,1115 b''
 # coding: utf-8
 """A tornado based IPython notebook server."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import base64
 import datetime
 import errno
 import importlib
 import io
 import json
 import logging
 import os
 import random
 import re
 import select
 import signal
 import socket
 import sys
 import threading
 import webbrowser
 
 
 # check for pyzmq
 from IPython.utils.zmqrelated import check_for_zmq
 check_for_zmq('13', 'IPython.html')
 
 from jinja2 import Environment, FileSystemLoader
 
 # Install the pyzmq ioloop. This has to be done before anything else from
 # tornado is imported.
 from zmq.eventloop import ioloop
 ioloop.install()
 
 # check for tornado 3.1.0
 msg = "The IPython Notebook requires tornado >= 4.0"
 try:
     import tornado
 except ImportError:
     raise ImportError(msg)
 try:
     version_info = tornado.version_info
 except AttributeError:
     raise ImportError(msg + ", but you have < 1.1.0")
 if version_info < (4,0):
     raise ImportError(msg + ", but you have %s" % tornado.version)
 
 from tornado import httpserver
 from tornado import web
 from tornado.log import LogFormatter, app_log, access_log, gen_log
 
 from IPython.html import (
     DEFAULT_STATIC_FILES_PATH,
     DEFAULT_TEMPLATE_PATH_LIST,
 )
 from .base.handlers import Template404
 from .log import log_request
 from .services.kernels.kernelmanager import MappingKernelManager
 from .services.config import ConfigManager
 from .services.contents.manager import ContentsManager
 from .services.contents.filemanager import FileContentsManager
 from .services.clusters.clustermanager import ClusterManager
 from .services.sessions.sessionmanager import SessionManager
 
 from .auth.login import LoginHandler
 from .auth.logout import LogoutHandler
 from .base.handlers import IPythonHandler, FileFindHandler
 
 from IPython.config import Config
 from IPython.config.application import catch_config_error, boolean_flag
 from IPython.core.application import (
     BaseIPythonApplication, base_flags, base_aliases,
 )
 from IPython.core.profiledir import ProfileDir
 from IPython.kernel import KernelManager
 from IPython.kernel.kernelspec import KernelSpecManager
-from IPython.kernel.zmq.session import default_secure, Session
+from IPython.kernel.zmq.session import Session
 from IPython.nbformat.sign import NotebookNotary
 from IPython.utils.importstring import import_item
 from IPython.utils import submodule
 from IPython.utils.process import check_pid
 from IPython.utils.traitlets import (
     Dict, Unicode, Integer, List, Bool, Bytes, Instance,
     TraitError, Type,
 )
 from IPython.utils import py3compat
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils.sysinfo import get_sys_info
 
 from .nbextensions import SYSTEM_NBEXTENSIONS_DIRS
 from .utils import url_path_join
 
 #-----------------------------------------------------------------------------
 # Module globals
 #-----------------------------------------------------------------------------
 
 _examples = """
 ipython notebook                       # start the notebook
 ipython notebook --profile=sympy       # use the sympy profile
 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
 """
 
 #-----------------------------------------------------------------------------
 # Helper functions
 #-----------------------------------------------------------------------------
 
 def random_ports(port, n):
     """Generate a list of n random ports near the given port.
 
     The first 5 ports will be sequential, and the remaining n-5 will be
     randomly selected in the range [port-2*n, port+2*n].
     """
     for i in range(min(5, n)):
         yield port + i
     for i in range(n-5):
         yield max(1, port + random.randint(-2*n, 2*n))
 
 def load_handlers(name):
     """Load the (URL pattern, handler) tuples for each component."""
     name = 'IPython.html.' + name
     mod = __import__(name, fromlist=['default_handlers'])
     return mod.default_handlers
 
 #-----------------------------------------------------------------------------
 # The Tornado web application
 #-----------------------------------------------------------------------------
 
 class NotebookWebApplication(web.Application):
 
     def __init__(self, ipython_app, kernel_manager, contents_manager,
                  cluster_manager, session_manager, kernel_spec_manager,
                  config_manager, log,
                  base_url, default_url, settings_overrides, jinja_env_options):
 
         settings = self.init_settings(
             ipython_app, kernel_manager, contents_manager, cluster_manager,
             session_manager, kernel_spec_manager, config_manager, log, base_url,
             default_url, settings_overrides, jinja_env_options)
         handlers = self.init_handlers(settings)
 
         super(NotebookWebApplication, self).__init__(handlers, **settings)
 
     def init_settings(self, ipython_app, kernel_manager, contents_manager,
                       cluster_manager, session_manager, kernel_spec_manager,
                       config_manager,
                       log, base_url, default_url, settings_overrides,
                       jinja_env_options=None):
 
         _template_path = settings_overrides.get(
             "template_path",
             ipython_app.template_file_path,
         )
         if isinstance(_template_path, str):
             _template_path = (_template_path,)
         template_path = [os.path.expanduser(path) for path in _template_path]
 
         jenv_opt = jinja_env_options if jinja_env_options else {}
         env = Environment(loader=FileSystemLoader(template_path), **jenv_opt)
         
         sys_info = get_sys_info()
         if sys_info['commit_source'] == 'repository':
             # don't cache (rely on 304) when working from master
             version_hash = ''
         else:
             # reset the cache on server restart
             version_hash = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
 
         settings = dict(
             # basics
             log_function=log_request,
             base_url=base_url,
             default_url=default_url,
             template_path=template_path,
             static_path=ipython_app.static_file_path,
             static_handler_class = FileFindHandler,
             static_url_prefix = url_path_join(base_url,'/static/'),
             static_handler_args = {
                 # don't cache custom.js
                 'no_cache_paths': [url_path_join(base_url, 'static', 'custom')],
             },
             version_hash=version_hash,
             
             # authentication
             cookie_secret=ipython_app.cookie_secret,
             login_url=url_path_join(base_url,'/login'),
             login_handler_class=ipython_app.login_handler_class,
             logout_handler_class=ipython_app.logout_handler_class,
             password=ipython_app.password,
 
             # managers
             kernel_manager=kernel_manager,
             contents_manager=contents_manager,
             cluster_manager=cluster_manager,
             session_manager=session_manager,
             kernel_spec_manager=kernel_spec_manager,
             config_manager=config_manager,
 
             # IPython stuff
             nbextensions_path=ipython_app.nbextensions_path,
             websocket_url=ipython_app.websocket_url,
             mathjax_url=ipython_app.mathjax_url,
             config=ipython_app.config,
             jinja2_env=env,
             terminals_available=False,  # Set later if terminals are available
         )
 
         # allow custom overrides for the tornado web app.
         settings.update(settings_overrides)
         return settings
 
     def init_handlers(self, settings):
         """Load the (URL pattern, handler) tuples for each component."""
         
         # Order matters. The first handler to match the URL will handle the request.
         handlers = []
         handlers.extend(load_handlers('tree.handlers'))
         handlers.extend([(r"/login", settings['login_handler_class'])])
         handlers.extend([(r"/logout", settings['logout_handler_class'])])
         handlers.extend(load_handlers('files.handlers'))
         handlers.extend(load_handlers('notebook.handlers'))
         handlers.extend(load_handlers('nbconvert.handlers'))
         handlers.extend(load_handlers('kernelspecs.handlers'))
         handlers.extend(load_handlers('edit.handlers'))
         handlers.extend(load_handlers('services.config.handlers'))
         handlers.extend(load_handlers('services.kernels.handlers'))
         handlers.extend(load_handlers('services.contents.handlers'))
         handlers.extend(load_handlers('services.clusters.handlers'))
         handlers.extend(load_handlers('services.sessions.handlers'))
         handlers.extend(load_handlers('services.nbconvert.handlers'))
         handlers.extend(load_handlers('services.kernelspecs.handlers'))
         handlers.extend(load_handlers('services.security.handlers'))
         handlers.append(
             (r"/nbextensions/(.*)", FileFindHandler, {
                 'path': settings['nbextensions_path'],
                 'no_cache_paths': ['/'], # don't cache anything in nbextensions
             }),
         )
         # register base handlers last
         handlers.extend(load_handlers('base.handlers'))
         # set the URL that will be redirected from `/`
         handlers.append(
             (r'/?', web.RedirectHandler, {
                 'url' : settings['default_url'],
                 'permanent': False, # want 302, not 301
             })
         )
         # prepend base_url onto the patterns that we match
         new_handlers = []
         for handler in handlers:
             pattern = url_path_join(settings['base_url'], handler[0])
             new_handler = tuple([pattern] + list(handler[1:]))
             new_handlers.append(new_handler)
         # add 404 on the end, which will catch everything that falls through
         new_handlers.append((r'(.*)', Template404))
         return new_handlers
 
 
 class NbserverListApp(BaseIPythonApplication):
     
     description="List currently running notebook servers in this profile."
     
     flags = dict(
         json=({'NbserverListApp': {'json': True}},
               "Produce machine-readable JSON output."),
     )
     
     json = Bool(False, config=True,
           help="If True, each line of output will be a JSON object with the "
                   "details from the server info file.")
 
     def start(self):
         if not self.json:
             print("Currently running servers:")
         for serverinfo in list_running_servers(self.profile):
             if self.json:
                 print(json.dumps(serverinfo))
             else:
                 print(serverinfo['url'], "::", serverinfo['notebook_dir'])
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = dict(base_flags)
 flags['no-browser']=(
     {'NotebookApp' : {'open_browser' : False}},
     "Don't open the notebook in a browser after startup."
 )
 flags['pylab']=(
     {'NotebookApp' : {'pylab' : 'warn'}},
     "DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib."
 )
 flags['no-mathjax']=(
     {'NotebookApp' : {'enable_mathjax' : False}},
     """Disable MathJax
     
     MathJax is the javascript library IPython uses to render math/LaTeX. It is
     very large, so you may want to disable it if you have a slow internet
     connection, or for offline use of the notebook.
     
     When disabled, equations etc. will appear as their untransformed TeX source.
     """
 )
 
 # Add notebook manager flags
 flags.update(boolean_flag('script', 'FileContentsManager.save_script',
                'DEPRECATED, IGNORED',
                'DEPRECATED, IGNORED'))
 
 aliases = dict(base_aliases)
 
 aliases.update({
     'ip': 'NotebookApp.ip',
     'port': 'NotebookApp.port',
     'port-retries': 'NotebookApp.port_retries',
     'transport': 'KernelManager.transport',
     'keyfile': 'NotebookApp.keyfile',
     'certfile': 'NotebookApp.certfile',
     'notebook-dir': 'NotebookApp.notebook_dir',
     'browser': 'NotebookApp.browser',
     'pylab': 'NotebookApp.pylab',
 })
 
 #-----------------------------------------------------------------------------
 # NotebookApp
 #-----------------------------------------------------------------------------
 
 class NotebookApp(BaseIPythonApplication):
 
     name = 'ipython-notebook'
     
     description = """
         The IPython HTML Notebook.
         
         This launches a Tornado based HTML Notebook Server that serves up an
         HTML5/Javascript Notebook client.
     """
     examples = _examples
     aliases = aliases
     flags = flags
     
     classes = [
         KernelManager, ProfileDir, Session, MappingKernelManager,
         ContentsManager, FileContentsManager, NotebookNotary,
         KernelSpecManager,
     ]
     flags = Dict(flags)
     aliases = Dict(aliases)
     
     subcommands = dict(
         list=(NbserverListApp, NbserverListApp.description.splitlines()[0]),
     )
 
     ipython_kernel_argv = List(Unicode)
     
     _log_formatter_cls = LogFormatter
 
     def _log_level_default(self):
         return logging.INFO
 
     def _log_datefmt_default(self):
         """Exclude date from default date format"""
         return "%H:%M:%S"
     
     def _log_format_default(self):
         """override default log format to include time"""
         return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s"
 
     # create requested profiles by default, if they don't exist:
     auto_create = Bool(True)
 
     # file to be opened in the notebook server
     file_to_run = Unicode('', config=True)
 
     # Network related information
     
     allow_origin = Unicode('', config=True,
         help="""Set the Access-Control-Allow-Origin header
         
         Use '*' to allow any origin to access your server.
         
         Takes precedence over allow_origin_pat.
         """
     )
     
     allow_origin_pat = Unicode('', config=True,
         help="""Use a regular expression for the Access-Control-Allow-Origin header
         
         Requests from an origin matching the expression will get replies with:
         
             Access-Control-Allow-Origin: origin
         
         where `origin` is the origin of the request.
         
         Ignored if allow_origin is set.
         """
     )
     
     allow_credentials = Bool(False, config=True,
         help="Set the Access-Control-Allow-Credentials: true header"
     )
     
     default_url = Unicode('/tree', config=True,
         help="The default URL to redirect to from `/`"
     )
     
     ip = Unicode('localhost', config=True,
         help="The IP address the notebook server will listen on."
     )
 
     def _ip_changed(self, name, old, new):
         if new == u'*': self.ip = u''
 
     port = Integer(8888, config=True,
         help="The port the notebook server will listen on."
     )
     port_retries = Integer(50, config=True,
         help="The number of additional ports to try if the specified port is not available."
     )
 
     certfile = Unicode(u'', config=True, 
         help="""The full path to an SSL/TLS certificate file."""
     )
     
     keyfile = Unicode(u'', config=True, 
         help="""The full path to a private key file for usage with SSL/TLS."""
     )
     
     cookie_secret_file = Unicode(config=True,
         help="""The file where the cookie secret is stored."""
     )
     def _cookie_secret_file_default(self):
         if self.profile_dir is None:
             return ''
         return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret')
     
     cookie_secret = Bytes(b'', config=True,
         help="""The random bytes used to secure cookies.
         By default this is a new random number every time you start the Notebook.
         Set it to a value in a config file to enable logins to persist across server sessions.
         
         Note: Cookie secrets should be kept private, do not share config files with
         cookie_secret stored in plaintext (you can read the value from a file).
         """
     )
     def _cookie_secret_default(self):
         if os.path.exists(self.cookie_secret_file):
             with io.open(self.cookie_secret_file, 'rb') as f:
                 return f.read()
         else:
             secret = base64.encodestring(os.urandom(1024))
             self._write_cookie_secret_file(secret)
             return secret
     
     def _write_cookie_secret_file(self, secret):
         """write my secret to my secret_file"""
         self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file)
         with io.open(self.cookie_secret_file, 'wb') as f:
             f.write(secret)
         try:
             os.chmod(self.cookie_secret_file, 0o600)
         except OSError:
             self.log.warn(
                 "Could not set permissions on %s",
                 self.cookie_secret_file
             )
 
     password = Unicode(u'', config=True,
                       help="""Hashed password to use for web authentication.
 
                       To generate, type in a python/IPython shell:
 
                         from IPython.lib import passwd; passwd()
 
                       The string should be of the form type:salt:hashed-password.
                       """
     )
 
     open_browser = Bool(True, config=True,
                         help="""Whether to open in a browser after starting.
                         The specific browser used is platform dependent and
                         determined by the python standard library `webbrowser`
                         module, unless it is overridden using the --browser
                         (NotebookApp.browser) configuration option.
                         """)
 
     browser = Unicode(u'', config=True,
                       help="""Specify what command to use to invoke a web
                       browser when opening the notebook. If not specified, the
                       default browser will be determined by the `webbrowser`
                       standard library module, which allows setting of the
                       BROWSER environment variable to override it.
                       """)
     
     webapp_settings = Dict(config=True,
         help="DEPRECATED, use tornado_settings"
     )
     def _webapp_settings_changed(self, name, old, new):
         self.log.warn("\n    webapp_settings is deprecated, use tornado_settings.\n")
         self.tornado_settings = new
     
     tornado_settings = Dict(config=True,
             help="Supply overrides for the tornado.web.Application that the "
                  "IPython notebook uses.")
     
     ssl_options = Dict(config=True,
             help="""Supply SSL options for the tornado HTTPServer.
             See the tornado docs for details.""")
     
     jinja_environment_options = Dict(config=True, 
             help="Supply extra arguments that will be passed to Jinja environment.")
     
     enable_mathjax = Bool(True, config=True,
         help="""Whether to enable MathJax for typesetting math/TeX
 
         MathJax is the javascript library IPython uses to render math/LaTeX. It is
         very large, so you may want to disable it if you have a slow internet
         connection, or for offline use of the notebook.
 
         When disabled, equations etc. will appear as their untransformed TeX source.
         """
     )
     def _enable_mathjax_changed(self, name, old, new):
         """set mathjax url to empty if mathjax is disabled"""
         if not new:
             self.mathjax_url = u''
 
     base_url = Unicode('/', config=True,
                                help='''The base URL for the notebook server.
 
                                Leading and trailing slashes can be omitted,
                                and will automatically be added.
                                ''')
     def _base_url_changed(self, name, old, new):
         if not new.startswith('/'):
             self.base_url = '/'+new
         elif not new.endswith('/'):
             self.base_url = new+'/'
     
     base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""")
     def _base_project_url_changed(self, name, old, new):
         self.log.warn("base_project_url is deprecated, use base_url")
         self.base_url = new
 
     extra_static_paths = List(Unicode, config=True,
         help="""Extra paths to search for serving static files.
         
         This allows adding javascript/css to be available from the notebook server machine,
         or overriding individual files in the IPython"""
     )
     def _extra_static_paths_default(self):
         return [os.path.join(self.profile_dir.location, 'static')]
     
     @property
     def static_file_path(self):
         """return extra paths + the default location"""
         return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
 
     extra_template_paths = List(Unicode, config=True,
         help="""Extra paths to search for serving jinja templates.
 
         Can be used to override templates from IPython.html.templates."""
     )
     def _extra_template_paths_default(self):
         return []
 
     @property
     def template_file_path(self):
         """return extra paths + the default locations"""
         return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST
 
     extra_nbextensions_path = List(Unicode, config=True,
         help="""extra paths to look for Javascript notebook extensions"""
     )
     
     @property
     def nbextensions_path(self):
         """The path to look for Javascript notebook extensions"""
         return self.extra_nbextensions_path + [os.path.join(get_ipython_dir(), 'nbextensions')] + SYSTEM_NBEXTENSIONS_DIRS
 
     websocket_url = Unicode("", config=True,
         help="""The base URL for websockets,
         if it differs from the HTTP server (hint: it almost certainly doesn't).
         
         Should be in the form of an HTTP origin: ws[s]://hostname[:port]
         """
     )
     mathjax_url = Unicode("", config=True,
         help="""The url for MathJax.js."""
     )
     def _mathjax_url_default(self):
         if not self.enable_mathjax:
             return u''
         static_url_prefix = self.tornado_settings.get("static_url_prefix",
                          url_path_join(self.base_url, "static")
         )
         
         # try local mathjax, either in nbextensions/mathjax or static/mathjax
         for (url_prefix, search_path) in [
             (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path),
             (static_url_prefix, self.static_file_path),
         ]:
             self.log.debug("searching for local mathjax in %s", search_path)
             try:
                 mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path)
             except IOError:
                 continue
             else:
                 url = url_path_join(url_prefix, u"mathjax/MathJax.js")
                 self.log.info("Serving local MathJax from %s at %s", mathjax, url)
                 return url
         
         # no local mathjax, serve from CDN
         url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js"
         self.log.info("Using MathJax from CDN: %s", url)
         return url
     
     def _mathjax_url_changed(self, name, old, new):
         if new and not self.enable_mathjax:
             # enable_mathjax=False overrides mathjax_url
             self.mathjax_url = u''
         else:
             self.log.info("Using MathJax: %s", new)
 
     contents_manager_class = Type(
         default_value=FileContentsManager,
         klass=ContentsManager,
         config=True,
         help='The notebook manager class to use.'
     )
     kernel_manager_class = Type(
         default_value=MappingKernelManager,
         config=True,
         help='The kernel manager class to use.'
     )
     session_manager_class = Type(
         default_value=SessionManager,
         config=True,
         help='The session manager class to use.'
     )
     cluster_manager_class = Type(
         default_value=ClusterManager,
         config=True,
         help='The cluster manager class to use.'
     )
 
     config_manager_class = Type(
         default_value=ConfigManager,
         config = True,
         help='The config manager class to use'
     )
 
     kernel_spec_manager = Instance(KernelSpecManager)
 
     kernel_spec_manager_class = Type(
         default_value=KernelSpecManager,
         config=True,
         help="""
         The kernel spec manager class to use. Should be a subclass
         of `IPython.kernel.kernelspec.KernelSpecManager`.
 
         The Api of KernelSpecManager is provisional and might change
         without warning between this version of IPython and the next stable one.
         """
     )
 
     login_handler_class = Type(
         default_value=LoginHandler,
         klass=web.RequestHandler,
         config=True,
         help='The login handler class to use.',
     )
 
     logout_handler_class = Type(
         default_value=LogoutHandler,
         klass=web.RequestHandler,
         config=True,
         help='The logout handler class to use.',
     )
 
     trust_xheaders = Bool(False, config=True,
         help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
               "sent by the upstream reverse proxy. Necessary if the proxy handles SSL")
     )
 
     info_file = Unicode()
 
     def _info_file_default(self):
         info_file = "nbserver-%s.json"%os.getpid()
         return os.path.join(self.profile_dir.security_dir, info_file)
     
     pylab = Unicode('disabled', config=True,
         help="""
         DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.
         """
     )
     def _pylab_changed(self, name, old, new):
         """when --pylab is specified, display a warning and exit"""
         if new != 'warn':
             backend = ' %s' % new
         else:
             backend = ''
         self.log.error("Support for specifying --pylab on the command line has been removed.")
         self.log.error(
             "Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself.".format(backend)
         )
         self.exit(1)
 
     notebook_dir = Unicode(config=True,
         help="The directory to use for notebooks and kernels."
     )
 
     def _notebook_dir_default(self):
         if self.file_to_run:
             return os.path.dirname(os.path.abspath(self.file_to_run))
         else:
             return py3compat.getcwd()
 
     def _notebook_dir_changed(self, name, old, new):
         """Do a bit of validation of the notebook dir."""
         if not os.path.isabs(new):
             # If we receive a non-absolute path, make it absolute.
             self.notebook_dir = os.path.abspath(new)
             return
         if not os.path.isdir(new):
             raise TraitError("No such notebook dir: %r" % new)
         
         # setting App.notebook_dir implies setting notebook and kernel dirs as well
         self.config.FileContentsManager.root_dir = new
         self.config.MappingKernelManager.root_dir = new
 
     server_extensions = List(Unicode(), config=True,
         help=("Python modules to load as notebook server extensions. "
               "This is an experimental API, and may change in future releases.")
     )
 
     def parse_command_line(self, argv=None):
         super(NotebookApp, self).parse_command_line(argv)
         
         if self.extra_args:
             arg0 = self.extra_args[0]
             f = os.path.abspath(arg0)
             self.argv.remove(arg0)
             if not os.path.exists(f):
                 self.log.critical("No such file or directory: %s", f)
                 self.exit(1)
             
             # Use config here, to ensure that it takes higher priority than
             # anything that comes from the profile.
             c = Config()
             if os.path.isdir(f):
                 c.NotebookApp.notebook_dir = f
             elif os.path.isfile(f):
                 c.NotebookApp.file_to_run = f
             self.update_config(c)
 
     def init_kernel_argv(self):
         """add the profile-dir to arguments to be passed to IPython kernels"""
         # FIXME: remove special treatment of IPython kernels
         # Kernel should get *absolute* path to profile directory
         self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location]
 
     def init_configurables(self):
-        # force Session default to be secure
-        default_secure(self.config)
-
         self.kernel_spec_manager = self.kernel_spec_manager_class(
             parent=self,
             ipython_dir=self.ipython_dir,
         )
         self.kernel_manager = self.kernel_manager_class(
             parent=self,
             log=self.log,
             ipython_kernel_argv=self.ipython_kernel_argv,
             connection_dir=self.profile_dir.security_dir,
         )
         self.contents_manager = self.contents_manager_class(
             parent=self,
             log=self.log,
         )
         self.session_manager = self.session_manager_class(
             parent=self,
             log=self.log,
             kernel_manager=self.kernel_manager,
             contents_manager=self.contents_manager,
         )
         self.cluster_manager = self.cluster_manager_class(
             parent=self,
             log=self.log,
         )
 
         self.config_manager = self.config_manager_class(
             parent=self,
             log=self.log,
             profile_dir=self.profile_dir.location,
         )
 
     def init_logging(self):
         # This prevents double log messages because tornado use a root logger that
         # self.log is a child of. The logging module dipatches log messages to a log
         # and all of its ancenstors until propagate is set to False.
         self.log.propagate = False
         
         for log in app_log, access_log, gen_log:
             # consistent log output name (NotebookApp instead of tornado.access, etc.)
             log.name = self.log.name
         # hook up tornado 3's loggers to our app handlers
         logger = logging.getLogger('tornado')
         logger.propagate = True
         logger.parent = self.log
         logger.setLevel(self.log.level)
     
     def init_webapp(self):
         """initialize tornado webapp and httpserver"""
         self.tornado_settings['allow_origin'] = self.allow_origin
         if self.allow_origin_pat:
             self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat)
         self.tornado_settings['allow_credentials'] = self.allow_credentials
         # ensure default_url starts with base_url
         if not self.default_url.startswith(self.base_url):
             self.default_url = url_path_join(self.base_url, self.default_url)
         
         self.web_app = NotebookWebApplication(
             self, self.kernel_manager, self.contents_manager,
             self.cluster_manager, self.session_manager, self.kernel_spec_manager,
             self.config_manager,
             self.log, self.base_url, self.default_url, self.tornado_settings,
             self.jinja_environment_options
         )
         ssl_options = self.ssl_options
         if self.certfile:
             ssl_options['certfile'] = self.certfile
         if self.keyfile:
             ssl_options['keyfile'] = self.keyfile
         if not ssl_options:
             # None indicates no SSL config
             ssl_options = None
         self.login_handler_class.validate_security(self, ssl_options=ssl_options)
         self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options,
                                                  xheaders=self.trust_xheaders)
 
         success = None
         for port in random_ports(self.port, self.port_retries+1):
             try:
                 self.http_server.listen(port, self.ip)
             except socket.error as e:
                 if e.errno == errno.EADDRINUSE:
                     self.log.info('The port %i is already in use, trying another random port.' % port)
                     continue
                 elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)):
                     self.log.warn("Permission to listen on port %i denied" % port)
                     continue
                 else:
                     raise
             else:
                 self.port = port
                 success = True
                 break
         if not success:
             self.log.critical('ERROR: the notebook server could not be started because '
                               'no available port could be found.')
             self.exit(1)
     
     @property
     def display_url(self):
         ip = self.ip if self.ip else '[all ip addresses on your system]'
         return self._url(ip)
 
     @property
     def connection_url(self):
         ip = self.ip if self.ip else 'localhost'
         return self._url(ip)
 
     def _url(self, ip):
         proto = 'https' if self.certfile else 'http'
         return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url)
 
     def init_terminals(self):
         try:
             from .terminal import initialize
             initialize(self.web_app)
             self.web_app.settings['terminals_available'] = True
         except ImportError as e:
             log = self.log.debug if sys.platform == 'win32' else self.log.warn
             log("Terminals not available (error was %s)", e)
 
     def init_signal(self):
         if not sys.platform.startswith('win'):
             signal.signal(signal.SIGINT, self._handle_sigint)
         signal.signal(signal.SIGTERM, self._signal_stop)
         if hasattr(signal, 'SIGUSR1'):
             # Windows doesn't support SIGUSR1
             signal.signal(signal.SIGUSR1, self._signal_info)
         if hasattr(signal, 'SIGINFO'):
             # only on BSD-based systems
             signal.signal(signal.SIGINFO, self._signal_info)
     
     def _handle_sigint(self, sig, frame):
         """SIGINT handler spawns confirmation dialog"""
         # register more forceful signal handler for ^C^C case
         signal.signal(signal.SIGINT, self._signal_stop)
         # request confirmation dialog in bg thread, to avoid
         # blocking the App
         thread = threading.Thread(target=self._confirm_exit)
         thread.daemon = True
         thread.start()
     
     def _restore_sigint_handler(self):
         """callback for restoring original SIGINT handler"""
         signal.signal(signal.SIGINT, self._handle_sigint)
     
     def _confirm_exit(self):
         """confirm shutdown on ^C
         
         A second ^C, or answering 'y' within 5s will cause shutdown,
         otherwise original SIGINT handler will be restored.
         
         This doesn't work on Windows.
         """
         info = self.log.info
         info('interrupted')
         print(self.notebook_info())
         sys.stdout.write("Shutdown this notebook server (y/[n])? ")
         sys.stdout.flush()
         r,w,x = select.select([sys.stdin], [], [], 5)
         if r:
             line = sys.stdin.readline()
             if line.lower().startswith('y') and 'n' not in line.lower():
                 self.log.critical("Shutdown confirmed")
                 ioloop.IOLoop.current().stop()
                 return
         else:
             print("No answer for 5s:", end=' ')
         print("resuming operation...")
         # no answer, or answer is no:
         # set it back to original SIGINT handler
         # use IOLoop.add_callback because signal.signal must be called
         # from main thread
         ioloop.IOLoop.current().add_callback(self._restore_sigint_handler)
     
     def _signal_stop(self, sig, frame):
         self.log.critical("received signal %s, stopping", sig)
         ioloop.IOLoop.current().stop()
 
     def _signal_info(self, sig, frame):
         print(self.notebook_info())
     
     def init_components(self):
         """Check the components submodule, and warn if it's unclean"""
         status = submodule.check_submodule_status()
         if status == 'missing':
             self.log.warn("components submodule missing, running `git submodule update`")
             submodule.update_submodules(submodule.ipython_parent())
         elif status == 'unclean':
             self.log.warn("components submodule unclean, you may see 404s on static/components")
             self.log.warn("run `setup.py submodule` or `git submodule update` to update")
 
     def init_server_extensions(self):
         """Load any extensions specified by config.
 
         Import the module, then call the load_jupyter_server_extension function,
         if one exists.
         
         The extension API is experimental, and may change in future releases.
         """
         for modulename in self.server_extensions:
             try:
                 mod = importlib.import_module(modulename)
                 func = getattr(mod, 'load_jupyter_server_extension', None)
                 if func is not None:
                     func(self)
             except Exception:
                 self.log.warn("Error loading server extension %s", modulename,
                               exc_info=True)
     
     @catch_config_error
     def initialize(self, argv=None):
         super(NotebookApp, self).initialize(argv)
         self.init_logging()
         self.init_kernel_argv()
         self.init_configurables()
         self.init_components()
         self.init_webapp()
         self.init_terminals()
         self.init_signal()
         self.init_server_extensions()
 
     def cleanup_kernels(self):
         """Shutdown all kernels.
         
         The kernels will shutdown themselves when this process no longer exists,
         but explicit shutdown allows the KernelManagers to cleanup the connection files.
         """
         self.log.info('Shutting down kernels')
         self.kernel_manager.shutdown_all()
 
     def notebook_info(self):
         "Return the current working directory and the server url information"
         info = self.contents_manager.info_string() + "\n"
         info += "%d active kernels \n" % len(self.kernel_manager._kernels)
         return info + "The IPython Notebook is running at: %s" % self.display_url
 
     def server_info(self):
         """Return a JSONable dict of information about this server."""
         return {'url': self.connection_url,
                 'hostname': self.ip if self.ip else 'localhost',
                 'port': self.port,
                 'secure': bool(self.certfile),
                 'base_url': self.base_url,
                 'notebook_dir': os.path.abspath(self.notebook_dir),
                 'pid': os.getpid()
                }
 
     def write_server_info_file(self):
         """Write the result of server_info() to the JSON file info_file."""
         with open(self.info_file, 'w') as f:
             json.dump(self.server_info(), f, indent=2)
 
     def remove_server_info_file(self):
         """Remove the nbserver-<pid>.json file created for this server.
         
         Ignores the error raised when the file has already been removed.
         """
         try:
             os.unlink(self.info_file)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
     def start(self):
         """ Start the IPython Notebook server app, after initialization
         
         This method takes no arguments so all configuration and initialization
         must be done prior to calling this method."""
         if self.subapp is not None:
             return self.subapp.start()
 
         info = self.log.info
         for line in self.notebook_info().split("\n"):
             info(line)
         info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")
 
         self.write_server_info_file()
 
         if self.open_browser or self.file_to_run:
             try:
                 browser = webbrowser.get(self.browser or None)
             except webbrowser.Error as e:
                 self.log.warn('No web browser found: %s.' % e)
                 browser = None
             
             if self.file_to_run:
                 if not os.path.exists(self.file_to_run):
                     self.log.critical("%s does not exist" % self.file_to_run)
                     self.exit(1)
 
                 relpath = os.path.relpath(self.file_to_run, self.notebook_dir)
                 uri = url_path_join('notebooks', *relpath.split(os.sep))
             else:
                 uri = 'tree'
             if browser:
                 b = lambda : browser.open(url_path_join(self.connection_url, uri),
                                           new=2)
                 threading.Thread(target=b).start()
         
         self.io_loop = ioloop.IOLoop.current()
         if sys.platform.startswith('win'):
             # add no-op to wake every 5s
             # to handle signals that may be ignored by the inner loop
             pc = ioloop.PeriodicCallback(lambda : None, 5000)
             pc.start()
         try:
             self.io_loop.start()
         except KeyboardInterrupt:
             info("Interrupted...")
         finally:
             self.cleanup_kernels()
             self.remove_server_info_file()
     
     def stop(self):
         def _stop():
             self.http_server.stop()
             self.io_loop.stop()
         self.io_loop.add_callback(_stop)
 
 
 def list_running_servers(profile='default'):
     """Iterate over the server info files of running notebook servers.
     
     Given a profile name, find nbserver-* files in the security directory of
     that profile, and yield dicts of their information, each one pertaining to
     a currently running notebook server instance.
     """
     pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), name=profile)
     for file in os.listdir(pd.security_dir):
         if file.startswith('nbserver-'):
             with io.open(os.path.join(pd.security_dir, file), encoding='utf-8') as f:
                 info = json.load(f)
 
             # Simple check whether that process is really still running
             # Also remove leftover files from IPython 2.x without a pid field
             if ('pid' in info) and check_pid(info['pid']):
                 yield info
             else:
                 # If the process has died, try to delete its info file
                 try:
                     os.unlink(file)
                 except OSError:
                     pass  # TODO: This should warn or log or something
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 launch_new_instance = NotebookApp.launch_instance
 

IPython/html/services/kernels/handlers.py

@@ -1,286 +1,288 b''
 """Tornado handlers for kernels."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import json
 import logging
 from tornado import gen, web
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
 from IPython.utils.jsonutil import date_default
 from IPython.utils.py3compat import cast_unicode
 from IPython.html.utils import url_path_join, url_escape
 
 from ...base.handlers import IPythonHandler, json_errors
 from ...base.zmqhandlers import AuthenticatedZMQStreamHandler, deserialize_binary_message
 
 from IPython.core.release import kernel_protocol_version
 
 class MainKernelHandler(IPythonHandler):
 
     @web.authenticated
     @json_errors
     def get(self):
         km = self.kernel_manager
         self.finish(json.dumps(km.list_kernels()))
 
     @web.authenticated
     @json_errors
     def post(self):
         km = self.kernel_manager
         model = self.get_json_body()
         if model is None:
             model = {
                 'name': km.default_kernel_name
             }
         else:
             model.setdefault('name', km.default_kernel_name)
 
         kernel_id = km.start_kernel(kernel_name=model['name'])
         model = km.kernel_model(kernel_id)
         location = url_path_join(self.base_url, 'api', 'kernels', kernel_id)
         self.set_header('Location', url_escape(location))
         self.set_status(201)
         self.finish(json.dumps(model))
 
 
 class KernelHandler(IPythonHandler):
 
     SUPPORTED_METHODS = ('DELETE', 'GET')
 
     @web.authenticated
     @json_errors
     def get(self, kernel_id):
         km = self.kernel_manager
         km._check_kernel_id(kernel_id)
         model = km.kernel_model(kernel_id)
         self.finish(json.dumps(model))
 
     @web.authenticated
     @json_errors
     def delete(self, kernel_id):
         km = self.kernel_manager
         km.shutdown_kernel(kernel_id)
         self.set_status(204)
         self.finish()
 
 
 class KernelActionHandler(IPythonHandler):
 
     @web.authenticated
     @json_errors
     def post(self, kernel_id, action):
         km = self.kernel_manager
         if action == 'interrupt':
             km.interrupt_kernel(kernel_id)
             self.set_status(204)
         if action == 'restart':
             km.restart_kernel(kernel_id)
             model = km.kernel_model(kernel_id)
             self.set_header('Location', '{0}api/kernels/{1}'.format(self.base_url, kernel_id))
             self.write(json.dumps(model))
         self.finish()
 
 
 class ZMQChannelsHandler(AuthenticatedZMQStreamHandler):
     
     @property
     def kernel_info_timeout(self):
         return self.settings.get('kernel_info_timeout', 10)
     
     def __repr__(self):
         return "%s(%s)" % (self.__class__.__name__, getattr(self, 'kernel_id', 'uninitialized'))
     
     def create_stream(self):
         km = self.kernel_manager
         identity = self.session.bsession
         for channel in ('shell', 'iopub', 'stdin'):
             meth = getattr(km, 'connect_' + channel)
             self.channels[channel] = stream = meth(self.kernel_id, identity=identity)
             stream.channel = channel
         km.add_restart_callback(self.kernel_id, self.on_kernel_restarted)
         km.add_restart_callback(self.kernel_id, self.on_restart_failed, 'dead')
     
     def request_kernel_info(self):
         """send a request for kernel_info"""
         km = self.kernel_manager
         kernel = km.get_kernel(self.kernel_id)
         try:
             # check for previous request
             future = kernel._kernel_info_future
         except AttributeError:
             self.log.debug("Requesting kernel info from %s", self.kernel_id)
             # Create a kernel_info channel to query the kernel protocol version.
             # This channel will be closed after the kernel_info reply is received.
             if self.kernel_info_channel is None:
                 self.kernel_info_channel = km.connect_shell(self.kernel_id)
             self.kernel_info_channel.on_recv(self._handle_kernel_info_reply)
             self.session.send(self.kernel_info_channel, "kernel_info_request")
             # store the future on the kernel, so only one request is sent
             kernel._kernel_info_future = self._kernel_info_future
         else:
             if not future.done():
                 self.log.debug("Waiting for pending kernel_info request")
             future.add_done_callback(lambda f: self._finish_kernel_info(f.result()))
         return self._kernel_info_future
     
     def _handle_kernel_info_reply(self, msg):
         """process the kernel_info_reply
         
         enabling msg spec adaptation, if necessary
         """
         idents,msg = self.session.feed_identities(msg)
         try:
             msg = self.session.deserialize(msg)
         except:
             self.log.error("Bad kernel_info reply", exc_info=True)
             self._kernel_info_future.set_result({})
             return
         else:
             info = msg['content']
             self.log.debug("Received kernel info: %s", info)
             if msg['msg_type'] != 'kernel_info_reply' or 'protocol_version' not in info:
                 self.log.error("Kernel info request failed, assuming current %s", info)
                 info = {}
             self._finish_kernel_info(info)
         
         # close the kernel_info channel, we don't need it anymore
         if self.kernel_info_channel:
             self.kernel_info_channel.close()
         self.kernel_info_channel = None
     
     def _finish_kernel_info(self, info):
         """Finish handling kernel_info reply
         
         Set up protocol adaptation, if needed,
         and signal that connection can continue.
         """
         protocol_version = info.get('protocol_version', kernel_protocol_version)
         if protocol_version != kernel_protocol_version:
             self.session.adapt_version = int(protocol_version.split('.')[0])
             self.log.info("Adapting to protocol v%s for kernel %s", protocol_version, self.kernel_id)
         if not self._kernel_info_future.done():
             self._kernel_info_future.set_result(info)
     
     def initialize(self):
         super(ZMQChannelsHandler, self).initialize()
         self.zmq_stream = None
         self.channels = {}
         self.kernel_id = None
         self.kernel_info_channel = None
         self._kernel_info_future = Future()
     
     @gen.coroutine
     def pre_get(self):
         # authenticate first
         super(ZMQChannelsHandler, self).pre_get()
         # then request kernel info, waiting up to a certain time before giving up.
         # We don't want to wait forever, because browsers don't take it well when
         # servers never respond to websocket connection requests.
+        kernel = self.kernel_manager.get_kernel(self.kernel_id)
+        self.session.key = kernel.session.key
         future = self.request_kernel_info()
         
         def give_up():
             """Don't wait forever for the kernel to reply"""
             if future.done():
                 return
             self.log.warn("Timeout waiting for kernel_info reply from %s", self.kernel_id)
             future.set_result({})
         loop = IOLoop.current()
         loop.add_timeout(loop.time() + self.kernel_info_timeout, give_up)
         # actually wait for it
         yield future
     
     @gen.coroutine
     def get(self, kernel_id):
         self.kernel_id = cast_unicode(kernel_id, 'ascii')
         yield super(ZMQChannelsHandler, self).get(kernel_id=kernel_id)
     
     def open(self, kernel_id):
         super(ZMQChannelsHandler, self).open()
         try:
             self.create_stream()
         except web.HTTPError as e:
             self.log.error("Error opening stream: %s", e)
             # WebSockets don't response to traditional error codes so we
             # close the connection.
             for channel, stream in self.channels.items():
                 if not stream.closed():
                     stream.close()
             self.close()
         else:
             for channel, stream in self.channels.items():
                 stream.on_recv_stream(self._on_zmq_reply)
 
     def on_message(self, msg):
         if not self.channels:
             # already closed, ignore the message
             self.log.debug("Received message on closed websocket %r", msg)
             return
         if isinstance(msg, bytes):
             msg = deserialize_binary_message(msg)
         else:
             msg = json.loads(msg)
         channel = msg.pop('channel', None)
         if channel is None:
             self.log.warn("No channel specified, assuming shell: %s", msg)
             channel = 'shell'
         if channel not in self.channels:
             self.log.warn("No such channel: %r", channel)
             return
         stream = self.channels[channel]
         self.session.send(stream, msg)
 
     def on_close(self):
         km = self.kernel_manager
         if self.kernel_id in km:
             km.remove_restart_callback(
                 self.kernel_id, self.on_kernel_restarted,
             )
             km.remove_restart_callback(
                 self.kernel_id, self.on_restart_failed, 'dead',
             )
         # This method can be called twice, once by self.kernel_died and once
         # from the WebSocket close event. If the WebSocket connection is
         # closed before the ZMQ streams are setup, they could be None.
         for channel, stream in self.channels.items():
             if stream is not None and not stream.closed():
                 stream.on_recv(None)
                 # close the socket directly, don't wait for the stream
                 socket = stream.socket
                 stream.close()
                 socket.close()
         
         self.channels = {}
 
     def _send_status_message(self, status):
         msg = self.session.msg("status",
             {'execution_state': status}
         )
         msg['channel'] = 'iopub'
         self.write_message(json.dumps(msg, default=date_default))
 
     def on_kernel_restarted(self):
         logging.warn("kernel %s restarted", self.kernel_id)
         self._send_status_message('restarting')
 
     def on_restart_failed(self):
         logging.error("kernel %s restarted failed!", self.kernel_id)
         self._send_status_message('dead')
 
 
 #-----------------------------------------------------------------------------
 # URL to handler mappings
 #-----------------------------------------------------------------------------
 
 
 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
 _kernel_action_regex = r"(?P<action>restart|interrupt)"
 
 default_handlers = [
     (r"/api/kernels", MainKernelHandler),
     (r"/api/kernels/%s" % _kernel_id_regex, KernelHandler),
     (r"/api/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
     (r"/api/kernels/%s/channels" % _kernel_id_regex, ZMQChannelsHandler),
 ]

IPython/kernel/inprocess/ipkernel.py

@@ -1,172 +1,172 b''
 """An in-process kernel"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from contextlib import contextmanager
 import logging
 import sys
 
 from IPython.core.interactiveshell import InteractiveShellABC
 from IPython.utils.jsonutil import json_clean
 from IPython.utils.traitlets import Any, Enum, Instance, List, Type
 from IPython.kernel.zmq.ipkernel import IPythonKernel
 from IPython.kernel.zmq.zmqshell import ZMQInteractiveShell
 
 from .socket import DummySocket
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class InProcessKernel(IPythonKernel):
 
     #-------------------------------------------------------------------------
     # InProcessKernel interface
     #-------------------------------------------------------------------------
 
     # The frontends connected to this kernel.
     frontends = List(
         Instance('IPython.kernel.inprocess.client.InProcessKernelClient')
     )
 
     # The GUI environment that the kernel is running under. This need not be
     # specified for the normal operation for the kernel, but is required for
     # IPython's GUI support (including pylab). The default is 'inline' because
     # it is safe under all GUI toolkits.
     gui = Enum(('tk', 'gtk', 'wx', 'qt', 'qt4', 'inline'),
                default_value='inline')
 
     raw_input_str = Any()
     stdout = Any()
     stderr = Any()
 
     #-------------------------------------------------------------------------
     # Kernel interface
     #-------------------------------------------------------------------------
 
     shell_class = Type()
     shell_streams = List()
     control_stream = Any()
     iopub_socket = Instance(DummySocket, ())
     stdin_socket = Instance(DummySocket, ())
 
     def __init__(self, **traits):
         # When an InteractiveShell is instantiated by our base class, it binds
         # the current values of sys.stdout and sys.stderr.
         with self._redirected_io():
             super(InProcessKernel, self).__init__(**traits)
 
         self.iopub_socket.on_trait_change(self._io_dispatch, 'message_sent')
         self.shell.kernel = self
 
     def execute_request(self, stream, ident, parent):
         """ Override for temporary IO redirection. """
         with self._redirected_io():
             super(InProcessKernel, self).execute_request(stream, ident, parent)
 
     def start(self):
         """ Override registration of dispatchers for streams. """
         self.shell.exit_now = False
 
     def _abort_queue(self, stream):
         """ The in-process kernel doesn't abort requests. """
         pass
 
     def _input_request(self, prompt, ident, parent, password=False):
         # Flush output before making the request.
         self.raw_input_str = None
         sys.stderr.flush()
         sys.stdout.flush()
 
         # Send the input request.
         content = json_clean(dict(prompt=prompt, password=password))
         msg = self.session.msg(u'input_request', content, parent)
         for frontend in self.frontends:
             if frontend.session.session == parent['header']['session']:
                 frontend.stdin_channel.call_handlers(msg)
                 break
         else:
             logging.error('No frontend found for raw_input request')
             return str()
 
         # Await a response.
         while self.raw_input_str is None:
             frontend.stdin_channel.process_events()
         return self.raw_input_str
 
     #-------------------------------------------------------------------------
     # Protected interface
     #-------------------------------------------------------------------------
 
     @contextmanager
     def _redirected_io(self):
         """ Temporarily redirect IO to the kernel.
         """
         sys_stdout, sys_stderr = sys.stdout, sys.stderr
         sys.stdout, sys.stderr = self.stdout, self.stderr
         yield
         sys.stdout, sys.stderr = sys_stdout, sys_stderr
 
     #------ Trait change handlers --------------------------------------------
 
     def _io_dispatch(self):
         """ Called when a message is sent to the IO socket.
         """
         ident, msg = self.session.recv(self.iopub_socket, copy=False)
         for frontend in self.frontends:
             frontend.iopub_channel.call_handlers(msg)
         
     #------ Trait initializers -----------------------------------------------
 
     def _log_default(self):
         return logging.getLogger(__name__)
 
     def _session_default(self):
         from IPython.kernel.zmq.session import Session
-        return Session(parent=self)
+        return Session(parent=self, key=b'')
 
     def _shell_class_default(self):
         return InProcessInteractiveShell
 
     def _stdout_default(self):
         from IPython.kernel.zmq.iostream import OutStream
         return OutStream(self.session, self.iopub_socket, u'stdout', pipe=False)
 
     def _stderr_default(self):
         from IPython.kernel.zmq.iostream import OutStream
         return OutStream(self.session, self.iopub_socket, u'stderr', pipe=False)
 
 #-----------------------------------------------------------------------------
 # Interactive shell subclass
 #-----------------------------------------------------------------------------
 
 class InProcessInteractiveShell(ZMQInteractiveShell):
 
     kernel = Instance('IPython.kernel.inprocess.ipkernel.InProcessKernel')
 
     #-------------------------------------------------------------------------
     # InteractiveShell interface
     #-------------------------------------------------------------------------
 
     def enable_gui(self, gui=None):
         """Enable GUI integration for the kernel."""
         from IPython.kernel.zmq.eventloops import enable_gui
         if not gui:
             gui = self.kernel.gui
         return enable_gui(gui, kernel=self.kernel)
 
     def enable_matplotlib(self, gui=None):
         """Enable matplotlib integration for the kernel."""
         if not gui:
             gui = self.kernel.gui
         return super(InProcessInteractiveShell, self).enable_matplotlib(gui)
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
         """Activate pylab support at runtime."""
         if not gui:
             gui = self.kernel.gui
         return super(InProcessInteractiveShell, self).enable_pylab(gui, import_all,
                                                             welcome_message)
 
 InteractiveShellABC.register(InProcessInteractiveShell)

IPython/kernel/inprocess/manager.py

@@ -1,77 +1,71 b''
 """A kernel manager for in-process kernels."""
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2013  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 from IPython.utils.traitlets import Instance, DottedObjectName
 from IPython.kernel.managerabc import KernelManagerABC
 from IPython.kernel.manager import KernelManager
+from IPython.kernel.zmq.session import Session
 
-#-----------------------------------------------------------------------------
-# Main kernel manager class
-#-----------------------------------------------------------------------------
 
 class InProcessKernelManager(KernelManager):
     """A manager for an in-process kernel.
 
     This class implements the interface of
     `IPython.kernel.kernelmanagerabc.KernelManagerABC` and allows
     (asynchronous) frontends to be used seamlessly with an in-process kernel.
 
     See `IPython.kernel.kernelmanager.KernelManager` for docstrings.
     """
 
     # The kernel process with which the KernelManager is communicating.
     kernel = Instance('IPython.kernel.inprocess.ipkernel.InProcessKernel')
     # the client class for KM.client() shortcut
     client_class = DottedObjectName('IPython.kernel.inprocess.BlockingInProcessKernelClient')
-
+    
+    def _session_default(self):
+        # don't sign in-process messages
+        return Session(key=b'', parent=self)
+    
     #--------------------------------------------------------------------------
     # Kernel management methods
     #--------------------------------------------------------------------------
 
     def start_kernel(self, **kwds):
         from IPython.kernel.inprocess.ipkernel import InProcessKernel
-        self.kernel = InProcessKernel()
+        self.kernel = InProcessKernel(parent=self, session=self.session)
 
     def shutdown_kernel(self):
         self._kill_kernel()
 
     def restart_kernel(self, now=False, **kwds):
         self.shutdown_kernel()
         self.start_kernel(**kwds)
 
     @property
     def has_kernel(self):
         return self.kernel is not None
 
     def _kill_kernel(self):
         self.kernel = None
 
     def interrupt_kernel(self):
         raise NotImplementedError("Cannot interrupt in-process kernel.")
 
     def signal_kernel(self, signum):
         raise NotImplementedError("Cannot signal in-process kernel.")
 
     def is_alive(self):
         return self.kernel is not None
 
     def client(self, **kwargs):
         kwargs['kernel'] = self.kernel
         return super(InProcessKernelManager, self).client(**kwargs)
 
 
 #-----------------------------------------------------------------------------
 # ABC Registration
 #-----------------------------------------------------------------------------
 
 KernelManagerABC.register(InProcessKernelManager)

IPython/kernel/inprocess/tests/test_kernel.py

@@ -1,69 +1,69 b''
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import sys
 import unittest
 
 from IPython.kernel.inprocess.blocking import BlockingInProcessKernelClient
 from IPython.kernel.inprocess.manager import InProcessKernelManager
 from IPython.kernel.inprocess.ipkernel import InProcessKernel
 from IPython.kernel.tests.utils import assemble_output
 from IPython.testing.decorators import skipif_not_matplotlib
 from IPython.utils.io import capture_output
 from IPython.utils import py3compat
 
 if py3compat.PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 
 class InProcessKernelTestCase(unittest.TestCase):
 
     def setUp(self):
         self.km = InProcessKernelManager()
         self.km.start_kernel()
-        self.kc = BlockingInProcessKernelClient(kernel=self.km.kernel)
+        self.kc = self.km.client()
         self.kc.start_channels()
         self.kc.wait_for_ready()
 
     @skipif_not_matplotlib
     def test_pylab(self):
         """Does %pylab work in the in-process kernel?"""
         kc = self.kc
         kc.execute('%pylab')
         out, err = assemble_output(kc.iopub_channel)
         self.assertIn('matplotlib', out)
 
     def test_raw_input(self):
         """ Does the in-process kernel handle raw_input correctly?
         """
         io = StringIO('foobar\n')
         sys_stdin = sys.stdin
         sys.stdin = io
         try:
             if py3compat.PY3:
                 self.kc.execute('x = input()')
             else:
                 self.kc.execute('x = raw_input()')
         finally:
             sys.stdin = sys_stdin
         self.assertEqual(self.km.kernel.shell.user_ns.get('x'), 'foobar')
 
     def test_stdout(self):
         """ Does the in-process kernel correctly capture IO?
         """
         kernel = InProcessKernel()
 
         with capture_output() as io:
             kernel.shell.run_cell('print("foo")')
         self.assertEqual(io.stdout, 'foo\n')
 
-        kc = BlockingInProcessKernelClient(kernel=kernel)
+        kc = BlockingInProcessKernelClient(kernel=kernel, session=kernel.session)
         kernel.frontends.append(kc)
         kc.execute('print("bar")')
         out, err = assemble_output(kc.iopub_channel)
         self.assertEqual(out, 'bar\n')
 

IPython/kernel/inprocess/tests/test_kernelmanager.py

@@ -1,108 +1,108 b''
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import unittest
 
 from IPython.kernel.inprocess.blocking import BlockingInProcessKernelClient
 from IPython.kernel.inprocess.manager import InProcessKernelManager
 
 #-----------------------------------------------------------------------------
 # Test case
 #-----------------------------------------------------------------------------
 
 class InProcessKernelManagerTestCase(unittest.TestCase):
 
     def test_interface(self):
         """ Does the in-process kernel manager implement the basic KM interface?
         """
         km = InProcessKernelManager()
         self.assert_(not km.has_kernel)
 
         km.start_kernel()
         self.assert_(km.has_kernel)
         self.assert_(km.kernel is not None)
 
-        kc = BlockingInProcessKernelClient(kernel=km.kernel)
+        kc = km.client()
         self.assert_(not kc.channels_running)
 
         kc.start_channels()
         self.assert_(kc.channels_running)
 
         old_kernel = km.kernel
         km.restart_kernel()
         self.assertIsNotNone(km.kernel)
         self.assertNotEquals(km.kernel, old_kernel)
 
         km.shutdown_kernel()
         self.assert_(not km.has_kernel)
 
         self.assertRaises(NotImplementedError, km.interrupt_kernel)
         self.assertRaises(NotImplementedError, km.signal_kernel, 9)
 
         kc.stop_channels()
         self.assert_(not kc.channels_running)
 
     def test_execute(self):
         """ Does executing code in an in-process kernel work?
         """
         km = InProcessKernelManager()
         km.start_kernel()
-        kc = BlockingInProcessKernelClient(kernel=km.kernel)
+        kc = km.client()
         kc.start_channels()
         kc.wait_for_ready()
         kc.execute('foo = 1')
         self.assertEquals(km.kernel.shell.user_ns['foo'], 1)
 
     def test_complete(self):
         """ Does requesting completion from an in-process kernel work?
         """
         km = InProcessKernelManager()
         km.start_kernel()
-        kc = BlockingInProcessKernelClient(kernel=km.kernel)
+        kc = km.client()
         kc.start_channels()
         kc.wait_for_ready()
         km.kernel.shell.push({'my_bar': 0, 'my_baz': 1})
         kc.complete('my_ba', 5)
         msg = kc.get_shell_msg()
         self.assertEqual(msg['header']['msg_type'], 'complete_reply')
         self.assertEqual(sorted(msg['content']['matches']),
                           ['my_bar', 'my_baz'])
 
     def test_inspect(self):
         """ Does requesting object information from an in-process kernel work?
         """
         km = InProcessKernelManager()
         km.start_kernel()
-        kc = BlockingInProcessKernelClient(kernel=km.kernel)
+        kc = km.client()
         kc.start_channels()
         kc.wait_for_ready()
         km.kernel.shell.user_ns['foo'] = 1
         kc.inspect('foo')
         msg = kc.get_shell_msg()
         self.assertEqual(msg['header']['msg_type'], 'inspect_reply')
         content = msg['content']
         assert content['found']
         text = content['data']['text/plain']
         self.assertIn('int', text)
 
     def test_history(self):
         """ Does requesting history from an in-process kernel work?
         """
         km = InProcessKernelManager()
         km.start_kernel()
-        kc = BlockingInProcessKernelClient(kernel=km.kernel)
+        kc = km.client()
         kc.start_channels()
         kc.wait_for_ready()
         kc.execute('%who')
         kc.history(hist_access_type='tail', n=1)
         msg = kc.shell_channel.get_msgs()[-1]
         self.assertEquals(msg['header']['msg_type'], 'history_reply')
         history = msg['content']['history']
         self.assertEquals(len(history), 1)
         self.assertEquals(history[0][2], '%who')
 
 
 if __name__ == '__main__':
     unittest.main()

IPython/kernel/zmq/kernelapp.py

@@ -1,388 +1,387 b''
 """An Application for launching a kernel"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import atexit
 import os
 import sys
 import signal
 
 import zmq
 from zmq.eventloop import ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 
 from IPython.core.ultratb import FormattedTB
 from IPython.core.application import (
     BaseIPythonApplication, base_flags, base_aliases, catch_config_error
 )
 from IPython.core.profiledir import ProfileDir
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.utils import io
 from IPython.utils.path import filefind
 from IPython.utils.traitlets import (
     Any, Instance, Dict, Unicode, Integer, Bool, DottedObjectName, Type,
 )
 from IPython.utils.importstring import import_item
 from IPython.kernel import write_connection_file
 from IPython.kernel.connect import ConnectionFileMixin
 
 # local imports
 from .heartbeat import Heartbeat
 from .ipkernel import IPythonKernel
 from .parentpoller import ParentPollerUnix, ParentPollerWindows
 from .session import (
-    Session, session_flags, session_aliases, default_secure,
+    Session, session_flags, session_aliases,
 )
 from .zmqshell import ZMQInteractiveShell
 
 #-----------------------------------------------------------------------------
 # Flags and Aliases
 #-----------------------------------------------------------------------------
 
 kernel_aliases = dict(base_aliases)
 kernel_aliases.update({
     'ip' : 'IPKernelApp.ip',
     'hb' : 'IPKernelApp.hb_port',
     'shell' : 'IPKernelApp.shell_port',
     'iopub' : 'IPKernelApp.iopub_port',
     'stdin' : 'IPKernelApp.stdin_port',
     'control' : 'IPKernelApp.control_port',
     'f' : 'IPKernelApp.connection_file',
     'transport': 'IPKernelApp.transport',
 })
 
 kernel_flags = dict(base_flags)
 kernel_flags.update({
     'no-stdout' : (
             {'IPKernelApp' : {'no_stdout' : True}},
             "redirect stdout to the null device"),
     'no-stderr' : (
             {'IPKernelApp' : {'no_stderr' : True}},
             "redirect stderr to the null device"),
     'pylab' : (
         {'IPKernelApp' : {'pylab' : 'auto'}},
         """Pre-load matplotlib and numpy for interactive use with
         the default matplotlib backend."""),
 })
 
 # inherit flags&aliases for any IPython shell apps
 kernel_aliases.update(shell_aliases)
 kernel_flags.update(shell_flags)
 
 # inherit flags&aliases for Sessions
 kernel_aliases.update(session_aliases)
 kernel_flags.update(session_flags)
 
 _ctrl_c_message = """\
 NOTE: When using the `ipython kernel` entry point, Ctrl-C will not work.
 
 To exit, you will have to explicitly quit this process, by either sending
 "quit" from a client, or using Ctrl-\\ in UNIX-like environments.
 
 To read more about this, see https://github.com/ipython/ipython/issues/2049
 
 """
 
 #-----------------------------------------------------------------------------
 # Application class for starting an IPython Kernel
 #-----------------------------------------------------------------------------
 
 class IPKernelApp(BaseIPythonApplication, InteractiveShellApp,
         ConnectionFileMixin):
     name='ipython-kernel'
     aliases = Dict(kernel_aliases)
     flags = Dict(kernel_flags)
     classes = [IPythonKernel, ZMQInteractiveShell, ProfileDir, Session]
     # the kernel class, as an importstring
     kernel_class = Type('IPython.kernel.zmq.ipkernel.IPythonKernel', config=True,
                         klass='IPython.kernel.zmq.kernelbase.Kernel',
     help="""The Kernel subclass to be used.
     
     This should allow easy re-use of the IPKernelApp entry point
     to configure and launch kernels other than IPython's own.
     """)
     kernel = Any()
     poller = Any() # don't restrict this even though current pollers are all Threads
     heartbeat = Instance(Heartbeat)
     ports = Dict()
     
     # connection info:
     
     @property
     def abs_connection_file(self):
         if os.path.basename(self.connection_file) == self.connection_file:
             return os.path.join(self.profile_dir.security_dir, self.connection_file)
         else:
             return self.connection_file
         
 
     # streams, etc.
     no_stdout = Bool(False, config=True, help="redirect stdout to the null device")
     no_stderr = Bool(False, config=True, help="redirect stderr to the null device")
     outstream_class = DottedObjectName('IPython.kernel.zmq.iostream.OutStream',
         config=True, help="The importstring for the OutStream factory")
     displayhook_class = DottedObjectName('IPython.kernel.zmq.displayhook.ZMQDisplayHook',
         config=True, help="The importstring for the DisplayHook factory")
 
     # polling
     parent_handle = Integer(int(os.environ.get('JPY_PARENT_PID') or 0), config=True,
         help="""kill this process if its parent dies.  On Windows, the argument
         specifies the HANDLE of the parent process, otherwise it is simply boolean.
         """)
     interrupt = Integer(int(os.environ.get('JPY_INTERRUPT_EVENT') or 0), config=True,
         help="""ONLY USED ON WINDOWS
         Interrupt this process when the parent is signaled.
         """)
 
     def init_crash_handler(self):
         # Install minimal exception handling
         sys.excepthook = FormattedTB(mode='Verbose', color_scheme='NoColor',
                                      ostream=sys.__stdout__)
 
     def init_poller(self):
         if sys.platform == 'win32':
             if self.interrupt or self.parent_handle:
                 self.poller = ParentPollerWindows(self.interrupt, self.parent_handle)
         elif self.parent_handle:
             self.poller = ParentPollerUnix()
 
     def _bind_socket(self, s, port):
         iface = '%s://%s' % (self.transport, self.ip)
         if self.transport == 'tcp':
             if port <= 0:
                 port = s.bind_to_random_port(iface)
             else:
                 s.bind("tcp://%s:%i" % (self.ip, port))
         elif self.transport == 'ipc':
             if port <= 0:
                 port = 1
                 path = "%s-%i" % (self.ip, port)
                 while os.path.exists(path):
                     port = port + 1
                     path = "%s-%i" % (self.ip, port)
             else:
                 path = "%s-%i" % (self.ip, port)
             s.bind("ipc://%s" % path)
         return port
 
     def write_connection_file(self):
         """write connection info to JSON file"""
         cf = self.abs_connection_file
         self.log.debug("Writing connection file: %s", cf)
         write_connection_file(cf, ip=self.ip, key=self.session.key, transport=self.transport,
         shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port,
         iopub_port=self.iopub_port, control_port=self.control_port)
     
     def cleanup_connection_file(self):
         cf = self.abs_connection_file
         self.log.debug("Cleaning up connection file: %s", cf)
         try:
             os.remove(cf)
         except (IOError, OSError):
             pass
         
         self.cleanup_ipc_files()
     
     def init_connection_file(self):
         if not self.connection_file:
             self.connection_file = "kernel-%s.json"%os.getpid()
         try:
             self.connection_file = filefind(self.connection_file, ['.', self.profile_dir.security_dir])
         except IOError:
             self.log.debug("Connection file not found: %s", self.connection_file)
             # This means I own it, so I will clean it up:
             atexit.register(self.cleanup_connection_file)
             return
         try:
             self.load_connection_file()
         except Exception:
             self.log.error("Failed to load connection file: %r", self.connection_file, exc_info=True)
             self.exit(1)
     
     def init_sockets(self):
         # Create a context, a session, and the kernel sockets.
         self.log.info("Starting the kernel at pid: %i", os.getpid())
         context = zmq.Context.instance()
         # Uncomment this to try closing the context.
         # atexit.register(context.term)
 
         self.shell_socket = context.socket(zmq.ROUTER)
         self.shell_socket.linger = 1000
         self.shell_port = self._bind_socket(self.shell_socket, self.shell_port)
         self.log.debug("shell ROUTER Channel on port: %i" % self.shell_port)
 
         self.iopub_socket = context.socket(zmq.PUB)
         self.iopub_socket.linger = 1000
         self.iopub_port = self._bind_socket(self.iopub_socket, self.iopub_port)
         self.log.debug("iopub PUB Channel on port: %i" % self.iopub_port)
 
         self.stdin_socket = context.socket(zmq.ROUTER)
         self.stdin_socket.linger = 1000
         self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port)
         self.log.debug("stdin ROUTER Channel on port: %i" % self.stdin_port)
 
         self.control_socket = context.socket(zmq.ROUTER)
         self.control_socket.linger = 1000
         self.control_port = self._bind_socket(self.control_socket, self.control_port)
         self.log.debug("control ROUTER Channel on port: %i" % self.control_port)
     
     def init_heartbeat(self):
         """start the heart beating"""
         # heartbeat doesn't share context, because it mustn't be blocked
         # by the GIL, which is accessed by libzmq when freeing zero-copy messages
         hb_ctx = zmq.Context()
         self.heartbeat = Heartbeat(hb_ctx, (self.transport, self.ip, self.hb_port))
         self.hb_port = self.heartbeat.port
         self.log.debug("Heartbeat REP Channel on port: %i" % self.hb_port)
         self.heartbeat.start()
     
     def log_connection_info(self):
         """display connection info, and store ports"""
         basename = os.path.basename(self.connection_file)
         if basename == self.connection_file or \
             os.path.dirname(self.connection_file) == self.profile_dir.security_dir:
             # use shortname
             tail = basename
             if self.profile != 'default':
                 tail += " --profile %s" % self.profile
         else:
             tail = self.connection_file
         lines = [
             "To connect another client to this kernel, use:",
             "    --existing %s" % tail,
         ]
         # log connection info
         # info-level, so often not shown.
         # frontends should use the %connect_info magic
         # to see the connection info
         for line in lines:
             self.log.info(line)
         # also raw print to the terminal if no parent_handle (`ipython kernel`)
         if not self.parent_handle:
             io.rprint(_ctrl_c_message)
             for line in lines:
                 io.rprint(line)
 
         self.ports = dict(shell=self.shell_port, iopub=self.iopub_port,
                                 stdin=self.stdin_port, hb=self.hb_port,
                                 control=self.control_port)
 
     def init_blackhole(self):
         """redirects stdout/stderr to devnull if necessary"""
         if self.no_stdout or self.no_stderr:
             blackhole = open(os.devnull, 'w')
             if self.no_stdout:
                 sys.stdout = sys.__stdout__ = blackhole
             if self.no_stderr:
                 sys.stderr = sys.__stderr__ = blackhole
     
     def init_io(self):
         """Redirect input streams and set a display hook."""
         if self.outstream_class:
             outstream_factory = import_item(str(self.outstream_class))
             sys.stdout = outstream_factory(self.session, self.iopub_socket, u'stdout')
             sys.stderr = outstream_factory(self.session, self.iopub_socket, u'stderr')
         if self.displayhook_class:
             displayhook_factory = import_item(str(self.displayhook_class))
             sys.displayhook = displayhook_factory(self.session, self.iopub_socket)
 
     def init_signal(self):
         signal.signal(signal.SIGINT, signal.SIG_IGN)
 
     def init_kernel(self):
         """Create the Kernel object itself"""
         shell_stream = ZMQStream(self.shell_socket)
         control_stream = ZMQStream(self.control_socket)
         
         kernel_factory = self.kernel_class.instance
 
         kernel = kernel_factory(parent=self, session=self.session,
                                 shell_streams=[shell_stream, control_stream],
                                 iopub_socket=self.iopub_socket,
                                 stdin_socket=self.stdin_socket,
                                 log=self.log,
                                 profile_dir=self.profile_dir,
                                 user_ns=self.user_ns,
         )
         kernel.record_ports(self.ports)
         self.kernel = kernel
 
     def init_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
 
         # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab`
         # to ensure that any exception is printed straight to stderr.
         # Normally _showtraceback associates the reply with an execution,
         # which means frontends will never draw it, as this exception
         # is not associated with any execute request.
 
         shell = self.shell
         _showtraceback = shell._showtraceback
         try:
             # replace error-sending traceback with stderr
             def print_tb(etype, evalue, stb):
                 print ("GUI event loop or pylab initialization failed",
                        file=io.stderr)
                 print (shell.InteractiveTB.stb2text(stb), file=io.stderr)
             shell._showtraceback = print_tb
             InteractiveShellApp.init_gui_pylab(self)
         finally:
             shell._showtraceback = _showtraceback
 
     def init_shell(self):
         self.shell = getattr(self.kernel, 'shell', None)
         if self.shell:
             self.shell.configurables.append(self)
 
     @catch_config_error
     def initialize(self, argv=None):
         super(IPKernelApp, self).initialize(argv)
-        default_secure(self.config)
         self.init_blackhole()
         self.init_connection_file()
         self.init_poller()
         self.init_sockets()
         self.init_heartbeat()
         # writing/displaying connection info must be *after* init_sockets/heartbeat
         self.log_connection_info()
         self.write_connection_file()
         self.init_io()
         self.init_signal()
         self.init_kernel()
         # shell init steps
         self.init_path()
         self.init_shell()
         if self.shell:
             self.init_gui_pylab()
             self.init_extensions()
             self.init_code()
         # flush stdout/stderr, so that anything written to these streams during
         # initialization do not get associated with the first execution request
         sys.stdout.flush()
         sys.stderr.flush()
 
     def start(self):
         if self.poller is not None:
             self.poller.start()
         self.kernel.start()
         try:
             ioloop.IOLoop.instance().start()
         except KeyboardInterrupt:
             pass
 
 launch_new_instance = IPKernelApp.launch_instance
 
 def main():
     """Run an IPKernel as an application"""
     app = IPKernelApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()

IPython/kernel/zmq/session.py

@@ -1,873 +1,877 b''
 """Session object for building, serializing, sending, and receiving messages in
 IPython. The Session object supports serialization, HMAC signatures, and
 metadata on messages.
 
 Also defined here are utilities for working with Sessions:
 * A SessionFactory to be used as a base class for configurables that work with
 Sessions.
 * A Message object for convenience that allows attribute-access to the msg dict.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import hashlib
 import hmac
 import logging
 import os
 import pprint
 import random
 import uuid
 import warnings
 from datetime import datetime
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
 try:
     # We are using compare_digest to limit the surface of timing attacks
     from hmac import compare_digest
 except ImportError:
     # Python < 2.7.7: When digests don't match no feedback is provided,
     # limiting the surface of attack
     def compare_digest(a,b): return a == b
 
 import zmq
 from zmq.utils import jsonapi
 from zmq.eventloop.ioloop import IOLoop
 from zmq.eventloop.zmqstream import ZMQStream
 
 from IPython.core.release import kernel_protocol_version
 from IPython.config.configurable import Configurable, LoggingConfigurable
 from IPython.utils import io
 from IPython.utils.importstring import import_item
 from IPython.utils.jsonutil import extract_dates, squash_dates, date_default
 from IPython.utils.py3compat import (str_to_bytes, str_to_unicode, unicode_type,
                                      iteritems)
 from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set,
                                         DottedObjectName, CUnicode, Dict, Integer,
                                         TraitError,
 )
 from IPython.utils.pickleutil import PICKLE_PROTOCOL
 from IPython.kernel.adapter import adapt
 from IPython.kernel.zmq.serialize import MAX_ITEMS, MAX_BYTES
 
 #-----------------------------------------------------------------------------
 # utility functions
 #-----------------------------------------------------------------------------
 
 def squash_unicode(obj):
     """coerce unicode back to bytestrings."""
     if isinstance(obj,dict):
         for key in obj.keys():
             obj[key] = squash_unicode(obj[key])
             if isinstance(key, unicode_type):
                 obj[squash_unicode(key)] = obj.pop(key)
     elif isinstance(obj, list):
         for i,v in enumerate(obj):
             obj[i] = squash_unicode(v)
     elif isinstance(obj, unicode_type):
         obj = obj.encode('utf8')
     return obj
 
 #-----------------------------------------------------------------------------
 # globals and defaults
 #-----------------------------------------------------------------------------
 
 # ISO8601-ify datetime objects
 # allow unicode
 # disallow nan, because it's not actually valid JSON
 json_packer = lambda obj: jsonapi.dumps(obj, default=date_default,
     ensure_ascii=False, allow_nan=False,
 )
 json_unpacker = lambda s: jsonapi.loads(s)
 
 pickle_packer = lambda o: pickle.dumps(squash_dates(o), PICKLE_PROTOCOL)
 pickle_unpacker = pickle.loads
 
 default_packer = json_packer
 default_unpacker = json_unpacker
 
 DELIM = b"<IDS|MSG>"
 # singleton dummy tracker, which will always report as done
 DONE = zmq.MessageTracker()
 
 #-----------------------------------------------------------------------------
 # Mixin tools for apps that use Sessions
 #-----------------------------------------------------------------------------
 
 session_aliases = dict(
     ident = 'Session.session',
     user = 'Session.username',
     keyfile = 'Session.keyfile',
 )
 
 session_flags  = {
     'secure' : ({'Session' : { 'key' : str_to_bytes(str(uuid.uuid4())),
                             'keyfile' : '' }},
         """Use HMAC digests for authentication of messages.
         Setting this flag will generate a new UUID to use as the HMAC key.
         """),
     'no-secure' : ({'Session' : { 'key' : b'', 'keyfile' : '' }},
         """Don't authenticate messages."""),
 }
 
 def default_secure(cfg):
     """Set the default behavior for a config environment to be secure.
     
     If Session.key/keyfile have not been set, set Session.key to
     a new random UUID.
     """
-    
+    warnings.warn("default_secure is deprecated", DeprecationWarning)
     if 'Session' in cfg:
         if 'key' in cfg.Session or 'keyfile' in cfg.Session:
             return
     # key/keyfile not specified, generate new UUID:
     cfg.Session.key = str_to_bytes(str(uuid.uuid4()))
 
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class SessionFactory(LoggingConfigurable):
     """The Base class for configurables that have a Session, Context, logger,
     and IOLoop.
     """
 
     logname = Unicode('')
     def _logname_changed(self, name, old, new):
         self.log = logging.getLogger(new)
 
     # not configurable:
     context = Instance('zmq.Context')
     def _context_default(self):
         return zmq.Context.instance()
 
     session = Instance('IPython.kernel.zmq.session.Session')
 
     loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
     def _loop_default(self):
         return IOLoop.instance()
 
     def __init__(self, **kwargs):
         super(SessionFactory, self).__init__(**kwargs)
 
         if self.session is None:
             # construct the session
             self.session = Session(**kwargs)
 
 
 class Message(object):
     """A simple message object that maps dict keys to attributes.
 
     A Message can be created from a dict and a dict from a Message instance
     simply by calling dict(msg_obj)."""
 
     def __init__(self, msg_dict):
         dct = self.__dict__
         for k, v in iteritems(dict(msg_dict)):
             if isinstance(v, dict):
                 v = Message(v)
             dct[k] = v
 
     # Having this iterator lets dict(msg_obj) work out of the box.
     def __iter__(self):
         return iter(iteritems(self.__dict__))
 
     def __repr__(self):
         return repr(self.__dict__)
 
     def __str__(self):
         return pprint.pformat(self.__dict__)
 
     def __contains__(self, k):
         return k in self.__dict__
 
     def __getitem__(self, k):
         return self.__dict__[k]
 
 
 def msg_header(msg_id, msg_type, username, session):
     date = datetime.now()
     version = kernel_protocol_version
     return locals()
 
 def extract_header(msg_or_header):
     """Given a message or header, return the header."""
     if not msg_or_header:
         return {}
     try:
         # See if msg_or_header is the entire message.
         h = msg_or_header['header']
     except KeyError:
         try:
             # See if msg_or_header is just the header
             h = msg_or_header['msg_id']
         except KeyError:
             raise
         else:
             h = msg_or_header
     if not isinstance(h, dict):
         h = dict(h)
     return h
 
 class Session(Configurable):
     """Object for handling serialization and sending of messages.
 
     The Session object handles building messages and sending them
     with ZMQ sockets or ZMQStream objects.  Objects can communicate with each
     other over the network via Session objects, and only need to work with the
     dict-based IPython message spec. The Session will handle
     serialization/deserialization, security, and metadata.
 
     Sessions support configurable serialization via packer/unpacker traits,
     and signing with HMAC digests via the key/keyfile traits.
 
     Parameters
     ----------
 
     debug : bool
         whether to trigger extra debugging statements
     packer/unpacker : str : 'json', 'pickle' or import_string
         importstrings for methods to serialize message parts.  If just
         'json' or 'pickle', predefined JSON and pickle packers will be used.
         Otherwise, the entire importstring must be used.
 
         The functions must accept at least valid JSON input, and output *bytes*.
 
         For example, to use msgpack:
         packer = 'msgpack.packb', unpacker='msgpack.unpackb'
     pack/unpack : callables
         You can also set the pack/unpack callables for serialization directly.
     session : bytes
         the ID of this Session object.  The default is to generate a new UUID.
     username : unicode
         username added to message headers.  The default is to ask the OS.
     key : bytes
         The key used to initialize an HMAC signature.  If unset, messages
         will not be signed or checked.
     keyfile : filepath
         The file containing a key.  If this is set, `key` will be initialized
         to the contents of the file.
 
     """
 
     debug=Bool(False, config=True, help="""Debug output in the Session""")
 
     packer = DottedObjectName('json',config=True,
             help="""The name of the packer for serializing messages.
             Should be one of 'json', 'pickle', or an import name
             for a custom callable serializer.""")
     def _packer_changed(self, name, old, new):
         if new.lower() == 'json':
             self.pack = json_packer
             self.unpack = json_unpacker
             self.unpacker = new
         elif new.lower() == 'pickle':
             self.pack = pickle_packer
             self.unpack = pickle_unpacker
             self.unpacker = new
         else:
             self.pack = import_item(str(new))
 
     unpacker = DottedObjectName('json', config=True,
         help="""The name of the unpacker for unserializing messages.
         Only used with custom functions for `packer`.""")
     def _unpacker_changed(self, name, old, new):
         if new.lower() == 'json':
             self.pack = json_packer
             self.unpack = json_unpacker
             self.packer = new
         elif new.lower() == 'pickle':
             self.pack = pickle_packer
             self.unpack = pickle_unpacker
             self.packer = new
         else:
             self.unpack = import_item(str(new))
 
     session = CUnicode(u'', config=True,
         help="""The UUID identifying this session.""")
     def _session_default(self):
         u = unicode_type(uuid.uuid4())
         self.bsession = u.encode('ascii')
         return u
 
     def _session_changed(self, name, old, new):
         self.bsession = self.session.encode('ascii')
 
     # bsession is the session as bytes
     bsession = CBytes(b'')
 
     username = Unicode(str_to_unicode(os.environ.get('USER', 'username')),
         help="""Username for the Session. Default is your system username.""",
         config=True)
 
     metadata = Dict({}, config=True,
         help="""Metadata dictionary, which serves as the default top-level metadata dict for each message.""")
     
     # if 0, no adapting to do.
     adapt_version = Integer(0)
 
     # message signature related traits:
     
-    key = CBytes(b'', config=True,
-        help="""execution key, for extra authentication.""")
+    key = CBytes(config=True,
+        help="""execution key, for signing messages.""")
+    def _key_default(self):
+        return str_to_bytes(str(uuid.uuid4()))
+    
     def _key_changed(self):
         self._new_auth()
     
     signature_scheme = Unicode('hmac-sha256', config=True,
         help="""The digest scheme used to construct the message signatures.
         Must have the form 'hmac-HASH'.""")
     def _signature_scheme_changed(self, name, old, new):
         if not new.startswith('hmac-'):
             raise TraitError("signature_scheme must start with 'hmac-', got %r" % new)
         hash_name = new.split('-', 1)[1]
         try:
             self.digest_mod = getattr(hashlib, hash_name)
         except AttributeError:
             raise TraitError("hashlib has no such attribute: %s" % hash_name)
         self._new_auth()
     
     digest_mod = Any()
     def _digest_mod_default(self):
         return hashlib.sha256
     
     auth = Instance(hmac.HMAC)
     
     def _new_auth(self):
         if self.key:
             self.auth = hmac.HMAC(self.key, digestmod=self.digest_mod)
         else:
             self.auth = None
     
     digest_history = Set()
     digest_history_size = Integer(2**16, config=True,
         help="""The maximum number of digests to remember.
         
         The digest history will be culled when it exceeds this value.
         """
     )
 
     keyfile = Unicode('', config=True,
         help="""path to file containing execution key.""")
     def _keyfile_changed(self, name, old, new):
         with open(new, 'rb') as f:
             self.key = f.read().strip()
 
     # for protecting against sends from forks
     pid = Integer()
     
     # serialization traits:
     
     pack = Any(default_packer) # the actual packer function
     def _pack_changed(self, name, old, new):
         if not callable(new):
             raise TypeError("packer must be callable, not %s"%type(new))
 
     unpack = Any(default_unpacker) # the actual packer function
     def _unpack_changed(self, name, old, new):
         # unpacker is not checked - it is assumed to be
         if not callable(new):
             raise TypeError("unpacker must be callable, not %s"%type(new))
     
     # thresholds:
     copy_threshold = Integer(2**16, config=True,
         help="Threshold (in bytes) beyond which a buffer should be sent without copying.")
     buffer_threshold = Integer(MAX_BYTES, config=True,
         help="Threshold (in bytes) beyond which an object's buffer should be extracted to avoid pickling.")
     item_threshold = Integer(MAX_ITEMS, config=True,
         help="""The maximum number of items for a container to be introspected for custom serialization.
         Containers larger than this are pickled outright.
         """
     )
 
     
     def __init__(self, **kwargs):
         """create a Session object
 
         Parameters
         ----------
 
         debug : bool
             whether to trigger extra debugging statements
         packer/unpacker : str : 'json', 'pickle' or import_string
             importstrings for methods to serialize message parts.  If just
             'json' or 'pickle', predefined JSON and pickle packers will be used.
             Otherwise, the entire importstring must be used.
 
             The functions must accept at least valid JSON input, and output
             *bytes*.
 
             For example, to use msgpack:
             packer = 'msgpack.packb', unpacker='msgpack.unpackb'
         pack/unpack : callables
             You can also set the pack/unpack callables for serialization
             directly.
         session : unicode (must be ascii)
             the ID of this Session object.  The default is to generate a new
             UUID.
         bsession : bytes
             The session as bytes
         username : unicode
             username added to message headers.  The default is to ask the OS.
         key : bytes
             The key used to initialize an HMAC signature.  If unset, messages
             will not be signed or checked.
         signature_scheme : str
             The message digest scheme. Currently must be of the form 'hmac-HASH',
             where 'HASH' is a hashing function available in Python's hashlib.
             The default is 'hmac-sha256'.
             This is ignored if 'key' is empty.
         keyfile : filepath
             The file containing a key.  If this is set, `key` will be
             initialized to the contents of the file.
         """
         super(Session, self).__init__(**kwargs)
         self._check_packers()
         self.none = self.pack({})
         # ensure self._session_default() if necessary, so bsession is defined:
         self.session
         self.pid = os.getpid()
+        self._new_auth()
 
     @property
     def msg_id(self):
         """always return new uuid"""
         return str(uuid.uuid4())
 
     def _check_packers(self):
         """check packers for datetime support."""
         pack = self.pack
         unpack = self.unpack
 
         # check simple serialization
         msg = dict(a=[1,'hi'])
         try:
             packed = pack(msg)
         except Exception as e:
             msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}"
             if self.packer == 'json':
                 jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
             else:
                 jsonmsg = ""
             raise ValueError(
                 msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg)
             )
 
         # ensure packed message is bytes
         if not isinstance(packed, bytes):
             raise ValueError("message packed to %r, but bytes are required"%type(packed))
 
         # check that unpack is pack's inverse
         try:
             unpacked = unpack(packed)
             assert unpacked == msg
         except Exception as e:
             msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}"
             if self.packer == 'json':
                 jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
             else:
                 jsonmsg = ""
             raise ValueError(
                 msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg)
             )
 
         # check datetime support
         msg = dict(t=datetime.now())
         try:
             unpacked = unpack(pack(msg))
             if isinstance(unpacked['t'], datetime):
                 raise ValueError("Shouldn't deserialize to datetime")
         except Exception:
             self.pack = lambda o: pack(squash_dates(o))
             self.unpack = lambda s: unpack(s)
 
     def msg_header(self, msg_type):
         return msg_header(self.msg_id, msg_type, self.username, self.session)
 
     def msg(self, msg_type, content=None, parent=None, header=None, metadata=None):
         """Return the nested message dict.
 
         This format is different from what is sent over the wire. The
         serialize/deserialize methods converts this nested message dict to the wire
         format, which is a list of message parts.
         """
         msg = {}
         header = self.msg_header(msg_type) if header is None else header
         msg['header'] = header
         msg['msg_id'] = header['msg_id']
         msg['msg_type'] = header['msg_type']
         msg['parent_header'] = {} if parent is None else extract_header(parent)
         msg['content'] = {} if content is None else content
         msg['metadata'] = self.metadata.copy()
         if metadata is not None:
             msg['metadata'].update(metadata)
         return msg
 
     def sign(self, msg_list):
         """Sign a message with HMAC digest. If no auth, return b''.
 
         Parameters
         ----------
         msg_list : list
             The [p_header,p_parent,p_content] part of the message list.
         """
         if self.auth is None:
             return b''
         h = self.auth.copy()
         for m in msg_list:
             h.update(m)
         return str_to_bytes(h.hexdigest())
 
     def serialize(self, msg, ident=None):
         """Serialize the message components to bytes.
 
         This is roughly the inverse of deserialize. The serialize/deserialize
         methods work with full message lists, whereas pack/unpack work with
         the individual message parts in the message list.
 
         Parameters
         ----------
         msg : dict or Message
             The next message dict as returned by the self.msg method.
 
         Returns
         -------
         msg_list : list
             The list of bytes objects to be sent with the format::
 
                 [ident1, ident2, ..., DELIM, HMAC, p_header, p_parent,
                  p_metadata, p_content, buffer1, buffer2, ...]
 
             In this list, the ``p_*`` entities are the packed or serialized
             versions, so if JSON is used, these are utf8 encoded JSON strings.
         """
         content = msg.get('content', {})
         if content is None:
             content = self.none
         elif isinstance(content, dict):
             content = self.pack(content)
         elif isinstance(content, bytes):
             # content is already packed, as in a relayed message
             pass
         elif isinstance(content, unicode_type):
             # should be bytes, but JSON often spits out unicode
             content = content.encode('utf8')
         else:
             raise TypeError("Content incorrect type: %s"%type(content))
 
         real_message = [self.pack(msg['header']),
                         self.pack(msg['parent_header']),
                         self.pack(msg['metadata']),
                         content,
         ]
 
         to_send = []
 
         if isinstance(ident, list):
             # accept list of idents
             to_send.extend(ident)
         elif ident is not None:
             to_send.append(ident)
         to_send.append(DELIM)
 
         signature = self.sign(real_message)
         to_send.append(signature)
 
         to_send.extend(real_message)
 
         return to_send
 
     def send(self, stream, msg_or_type, content=None, parent=None, ident=None,
              buffers=None, track=False, header=None, metadata=None):
         """Build and send a message via stream or socket.
 
         The message format used by this function internally is as follows:
 
         [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
          buffer1,buffer2,...]
 
         The serialize/deserialize methods convert the nested message dict into this
         format.
 
         Parameters
         ----------
 
         stream : zmq.Socket or ZMQStream
             The socket-like object used to send the data.
         msg_or_type : str or Message/dict
             Normally, msg_or_type will be a msg_type unless a message is being
             sent more than once. If a header is supplied, this can be set to
             None and the msg_type will be pulled from the header.
 
         content : dict or None
             The content of the message (ignored if msg_or_type is a message).
         header : dict or None
             The header dict for the message (ignored if msg_to_type is a message).
         parent : Message or dict or None
             The parent or parent header describing the parent of this message
             (ignored if msg_or_type is a message).
         ident : bytes or list of bytes
             The zmq.IDENTITY routing path.
         metadata : dict or None
             The metadata describing the message
         buffers : list or None
             The already-serialized buffers to be appended to the message.
         track : bool
             Whether to track.  Only for use with Sockets, because ZMQStream
             objects cannot track messages.
             
 
         Returns
         -------
         msg : dict
             The constructed message.
         """
         if not isinstance(stream, zmq.Socket):
             # ZMQStreams and dummy sockets do not support tracking.
             track = False
 
         if isinstance(msg_or_type, (Message, dict)):
             # We got a Message or message dict, not a msg_type so don't
             # build a new Message.
             msg = msg_or_type
             buffers = buffers or msg.get('buffers', [])
         else:
             msg = self.msg(msg_or_type, content=content, parent=parent,
                            header=header, metadata=metadata)
         if not os.getpid() == self.pid:
             io.rprint("WARNING: attempted to send message from fork")
             io.rprint(msg)
             return
         buffers = [] if buffers is None else buffers
         if self.adapt_version:
             msg = adapt(msg, self.adapt_version)
         to_send = self.serialize(msg, ident)
         to_send.extend(buffers)
         longest = max([ len(s) for s in to_send ])
         copy = (longest < self.copy_threshold)
         
         if buffers and track and not copy:
             # only really track when we are doing zero-copy buffers
             tracker = stream.send_multipart(to_send, copy=False, track=True)
         else:
             # use dummy tracker, which will be done immediately
             tracker = DONE
             stream.send_multipart(to_send, copy=copy)
 
         if self.debug:
             pprint.pprint(msg)
             pprint.pprint(to_send)
             pprint.pprint(buffers)
 
         msg['tracker'] = tracker
 
         return msg
 
     def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
         """Send a raw message via ident path.
 
         This method is used to send a already serialized message.
 
         Parameters
         ----------
         stream : ZMQStream or Socket
             The ZMQ stream or socket to use for sending the message.
         msg_list : list
             The serialized list of messages to send. This only includes the
             [p_header,p_parent,p_metadata,p_content,buffer1,buffer2,...] portion of
             the message.
         ident : ident or list
             A single ident or a list of idents to use in sending.
         """
         to_send = []
         if isinstance(ident, bytes):
             ident = [ident]
         if ident is not None:
             to_send.extend(ident)
 
         to_send.append(DELIM)
         to_send.append(self.sign(msg_list))
         to_send.extend(msg_list)
         stream.send_multipart(to_send, flags, copy=copy)
 
     def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
         """Receive and unpack a message.
 
         Parameters
         ----------
         socket : ZMQStream or Socket
             The socket or stream to use in receiving.
 
         Returns
         -------
         [idents], msg
             [idents] is a list of idents and msg is a nested message dict of
             same format as self.msg returns.
         """
         if isinstance(socket, ZMQStream):
             socket = socket.socket
         try:
             msg_list = socket.recv_multipart(mode, copy=copy)
         except zmq.ZMQError as e:
             if e.errno == zmq.EAGAIN:
                 # We can convert EAGAIN to None as we know in this case
                 # recv_multipart won't return None.
                 return None,None
             else:
                 raise
         # split multipart message into identity list and message dict
         # invalid large messages can cause very expensive string comparisons
         idents, msg_list = self.feed_identities(msg_list, copy)
         try:
             return idents, self.deserialize(msg_list, content=content, copy=copy)
         except Exception as e:
             # TODO: handle it
             raise e
 
     def feed_identities(self, msg_list, copy=True):
         """Split the identities from the rest of the message.
 
         Feed until DELIM is reached, then return the prefix as idents and
         remainder as msg_list. This is easily broken by setting an IDENT to DELIM,
         but that would be silly.
 
         Parameters
         ----------
         msg_list : a list of Message or bytes objects
             The message to be split.
         copy : bool
             flag determining whether the arguments are bytes or Messages
 
         Returns
         -------
         (idents, msg_list) : two lists
             idents will always be a list of bytes, each of which is a ZMQ
             identity. msg_list will be a list of bytes or zmq.Messages of the
             form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and
             should be unpackable/unserializable via self.deserialize at this
             point.
         """
         if copy:
             idx = msg_list.index(DELIM)
             return msg_list[:idx], msg_list[idx+1:]
         else:
             failed = True
             for idx,m in enumerate(msg_list):
                 if m.bytes == DELIM:
                     failed = False
                     break
             if failed:
                 raise ValueError("DELIM not in msg_list")
             idents, msg_list = msg_list[:idx], msg_list[idx+1:]
             return [m.bytes for m in idents], msg_list
 
     def _add_digest(self, signature):
         """add a digest to history to protect against replay attacks"""
         if self.digest_history_size == 0:
             # no history, never add digests
             return
         
         self.digest_history.add(signature)
         if len(self.digest_history) > self.digest_history_size:
             # threshold reached, cull 10%
             self._cull_digest_history()
     
     def _cull_digest_history(self):
         """cull the digest history
         
         Removes a randomly selected 10% of the digest history
         """
         current = len(self.digest_history)
         n_to_cull = max(int(current // 10), current - self.digest_history_size)
         if n_to_cull >= current:
             self.digest_history = set()
             return
         to_cull = random.sample(self.digest_history, n_to_cull)
         self.digest_history.difference_update(to_cull)
     
     def deserialize(self, msg_list, content=True, copy=True):
         """Unserialize a msg_list to a nested message dict.
 
         This is roughly the inverse of serialize. The serialize/deserialize
         methods work with full message lists, whereas pack/unpack work with
         the individual message parts in the message list.
 
         Parameters
         ----------
         msg_list : list of bytes or Message objects
             The list of message parts of the form [HMAC,p_header,p_parent,
             p_metadata,p_content,buffer1,buffer2,...].
         content : bool (True)
             Whether to unpack the content dict (True), or leave it packed
             (False).
         copy : bool (True)
             Whether to return the bytes (True), or the non-copying Message
             object in each place (False).
 
         Returns
         -------
         msg : dict
             The nested message dict with top-level keys [header, parent_header,
             content, buffers].
         """
         minlen = 5
         message = {}
         if not copy:
             for i in range(minlen):
                 msg_list[i] = msg_list[i].bytes
         if self.auth is not None:
             signature = msg_list[0]
             if not signature:
                 raise ValueError("Unsigned Message")
             if signature in self.digest_history:
                 raise ValueError("Duplicate Signature: %r" % signature)
             self._add_digest(signature)
             check = self.sign(msg_list[1:5])
             if not compare_digest(signature, check):
                 raise ValueError("Invalid Signature: %r" % signature)
         if not len(msg_list) >= minlen:
             raise TypeError("malformed message, must have at least %i elements"%minlen)
         header = self.unpack(msg_list[1])
         message['header'] = extract_dates(header)
         message['msg_id'] = header['msg_id']
         message['msg_type'] = header['msg_type']
         message['parent_header'] = extract_dates(self.unpack(msg_list[2]))
         message['metadata'] = self.unpack(msg_list[3])
         if content:
             message['content'] = self.unpack(msg_list[4])
         else:
             message['content'] = msg_list[4]
 
         message['buffers'] = msg_list[5:]
         # adapt to the current version
         return adapt(message)
     
     def unserialize(self, *args, **kwargs):
         warnings.warn(
             "Session.unserialize is deprecated. Use Session.deserialize.",
             DeprecationWarning,
         )
         return self.deserialize(*args, **kwargs)
 
 
 def test_msg2obj():
     am = dict(x=1)
     ao = Message(am)
     assert ao.x == am['x']
 
     am['y'] = dict(z=1)
     ao = Message(am)
     assert ao.y.z == am['y']['z']
 
     k1, k2 = 'y', 'z'
     assert ao[k1][k2] == am[k1][k2]
 
     am2 = dict(ao)
     assert am['x'] == am2['x']
     assert am['y']['z'] == am2['y']['z']
 

IPython/kernel/zmq/tests/test_session.py

@@ -1,318 +1,318 b''
-"""test building messages with streamsession"""
+"""test building messages with Session"""
 
-#-------------------------------------------------------------------------------
-#  Copyright (C) 2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-------------------------------------------------------------------------------
-
-#-------------------------------------------------------------------------------
-# Imports
-#-------------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
+import hmac
 import os
 import uuid
 from datetime import datetime
 
 import zmq
 
 from zmq.tests import BaseZMQTestCase
 from zmq.eventloop.zmqstream import ZMQStream
 
 from IPython.kernel.zmq import session as ss
 
 from IPython.testing.decorators import skipif, module_not_available
 from IPython.utils.py3compat import string_types
 from IPython.utils import jsonutil
 
 def _bad_packer(obj):
     raise TypeError("I don't work")
 
 def _bad_unpacker(bytes):
     raise TypeError("I don't work either")
 
 class SessionTestCase(BaseZMQTestCase):
 
     def setUp(self):
         BaseZMQTestCase.setUp(self)
         self.session = ss.Session()
 
 
 class TestSession(SessionTestCase):
 
     def test_msg(self):
         """message format"""
         msg = self.session.msg('execute')
         thekeys = set('header parent_header metadata content msg_type msg_id'.split())
         s = set(msg.keys())
         self.assertEqual(s, thekeys)
         self.assertTrue(isinstance(msg['content'],dict))
         self.assertTrue(isinstance(msg['metadata'],dict))
         self.assertTrue(isinstance(msg['header'],dict))
         self.assertTrue(isinstance(msg['parent_header'],dict))
         self.assertTrue(isinstance(msg['msg_id'],str))
         self.assertTrue(isinstance(msg['msg_type'],str))
         self.assertEqual(msg['header']['msg_type'], 'execute')
         self.assertEqual(msg['msg_type'], 'execute')
 
     def test_serialize(self):
         msg = self.session.msg('execute', content=dict(a=10, b=1.1))
         msg_list = self.session.serialize(msg, ident=b'foo')
         ident, msg_list = self.session.feed_identities(msg_list)
         new_msg = self.session.deserialize(msg_list)
         self.assertEqual(ident[0], b'foo')
         self.assertEqual(new_msg['msg_id'],msg['msg_id'])
         self.assertEqual(new_msg['msg_type'],msg['msg_type'])
         self.assertEqual(new_msg['header'],msg['header'])
         self.assertEqual(new_msg['content'],msg['content'])
         self.assertEqual(new_msg['parent_header'],msg['parent_header'])
         self.assertEqual(new_msg['metadata'],msg['metadata'])
         # ensure floats don't come out as Decimal:
         self.assertEqual(type(new_msg['content']['b']),type(new_msg['content']['b']))
 
+    def test_default_secure(self):
+        self.assertIsInstance(self.session.key, bytes)
+        self.assertIsInstance(self.session.auth, hmac.HMAC)
+
     def test_send(self):
         ctx = zmq.Context.instance()
         A = ctx.socket(zmq.PAIR)
         B = ctx.socket(zmq.PAIR)
         A.bind("inproc://test")
         B.connect("inproc://test")
 
         msg = self.session.msg('execute', content=dict(a=10))
         self.session.send(A, msg, ident=b'foo', buffers=[b'bar'])
         
         ident, msg_list = self.session.feed_identities(B.recv_multipart())
         new_msg = self.session.deserialize(msg_list)
         self.assertEqual(ident[0], b'foo')
         self.assertEqual(new_msg['msg_id'],msg['msg_id'])
         self.assertEqual(new_msg['msg_type'],msg['msg_type'])
         self.assertEqual(new_msg['header'],msg['header'])
         self.assertEqual(new_msg['content'],msg['content'])
         self.assertEqual(new_msg['parent_header'],msg['parent_header'])
         self.assertEqual(new_msg['metadata'],msg['metadata'])
         self.assertEqual(new_msg['buffers'],[b'bar'])
-
+        
         content = msg['content']
         header = msg['header']
+        header['date'] = datetime.now()
         parent = msg['parent_header']
         metadata = msg['metadata']
         msg_type = header['msg_type']
         self.session.send(A, None, content=content, parent=parent,
             header=header, metadata=metadata, ident=b'foo', buffers=[b'bar'])
         ident, msg_list = self.session.feed_identities(B.recv_multipart())
         new_msg = self.session.deserialize(msg_list)
         self.assertEqual(ident[0], b'foo')
         self.assertEqual(new_msg['msg_id'],msg['msg_id'])
         self.assertEqual(new_msg['msg_type'],msg['msg_type'])
         self.assertEqual(new_msg['header'],msg['header'])
         self.assertEqual(new_msg['content'],msg['content'])
         self.assertEqual(new_msg['metadata'],msg['metadata'])
         self.assertEqual(new_msg['parent_header'],msg['parent_header'])
         self.assertEqual(new_msg['buffers'],[b'bar'])
-
+        
+        header['date'] = datetime.now()
+        
         self.session.send(A, msg, ident=b'foo', buffers=[b'bar'])
         ident, new_msg = self.session.recv(B)
         self.assertEqual(ident[0], b'foo')
         self.assertEqual(new_msg['msg_id'],msg['msg_id'])
         self.assertEqual(new_msg['msg_type'],msg['msg_type'])
         self.assertEqual(new_msg['header'],msg['header'])
         self.assertEqual(new_msg['content'],msg['content'])
         self.assertEqual(new_msg['metadata'],msg['metadata'])
         self.assertEqual(new_msg['parent_header'],msg['parent_header'])
         self.assertEqual(new_msg['buffers'],[b'bar'])
 
         A.close()
         B.close()
         ctx.term()
 
     def test_args(self):
         """initialization arguments for Session"""
         s = self.session
         self.assertTrue(s.pack is ss.default_packer)
         self.assertTrue(s.unpack is ss.default_unpacker)
         self.assertEqual(s.username, os.environ.get('USER', u'username'))
 
         s = ss.Session()
         self.assertEqual(s.username, os.environ.get('USER', u'username'))
 
         self.assertRaises(TypeError, ss.Session, pack='hi')
         self.assertRaises(TypeError, ss.Session, unpack='hi')
         u = str(uuid.uuid4())
         s = ss.Session(username=u'carrot', session=u)
         self.assertEqual(s.session, u)
         self.assertEqual(s.username, u'carrot')
 
     def test_tracking(self):
         """test tracking messages"""
         a,b = self.create_bound_pair(zmq.PAIR, zmq.PAIR)
         s = self.session
         s.copy_threshold = 1
         stream = ZMQStream(a)
         msg = s.send(a, 'hello', track=False)
         self.assertTrue(msg['tracker'] is ss.DONE)
         msg = s.send(a, 'hello', track=True)
         self.assertTrue(isinstance(msg['tracker'], zmq.MessageTracker))
         M = zmq.Message(b'hi there', track=True)
         msg = s.send(a, 'hello', buffers=[M], track=True)
         t = msg['tracker']
         self.assertTrue(isinstance(t, zmq.MessageTracker))
         self.assertRaises(zmq.NotDone, t.wait, .1)
         del M
         t.wait(1) # this will raise
 
 
     def test_unique_msg_ids(self):
         """test that messages receive unique ids"""
         ids = set()
         for i in range(2**12):
             h = self.session.msg_header('test')
             msg_id = h['msg_id']
             self.assertTrue(msg_id not in ids)
             ids.add(msg_id)
 
     def test_feed_identities(self):
         """scrub the front for zmq IDENTITIES"""
         theids = "engine client other".split()
         content = dict(code='whoda',stuff=object())
         themsg = self.session.msg('execute',content=content)
         pmsg = theids
 
     def test_session_id(self):
         session = ss.Session()
         # get bs before us
         bs = session.bsession
         us = session.session
         self.assertEqual(us.encode('ascii'), bs)
         session = ss.Session()
         # get us before bs
         us = session.session
         bs = session.bsession
         self.assertEqual(us.encode('ascii'), bs)
         # change propagates:
         session.session = 'something else'
         bs = session.bsession
         us = session.session
         self.assertEqual(us.encode('ascii'), bs)
         session = ss.Session(session='stuff')
         # get us before bs
         self.assertEqual(session.bsession, session.session.encode('ascii'))
         self.assertEqual(b'stuff', session.bsession)
 
     def test_zero_digest_history(self):
         session = ss.Session(digest_history_size=0)
         for i in range(11):
             session._add_digest(uuid.uuid4().bytes)
         self.assertEqual(len(session.digest_history), 0)
 
     def test_cull_digest_history(self):
         session = ss.Session(digest_history_size=100)
         for i in range(100):
             session._add_digest(uuid.uuid4().bytes)
         self.assertTrue(len(session.digest_history) == 100)
         session._add_digest(uuid.uuid4().bytes)
         self.assertTrue(len(session.digest_history) == 91)
         for i in range(9):
             session._add_digest(uuid.uuid4().bytes)
         self.assertTrue(len(session.digest_history) == 100)
         session._add_digest(uuid.uuid4().bytes)
         self.assertTrue(len(session.digest_history) == 91)
     
     def test_bad_pack(self):
         try:
             session = ss.Session(pack=_bad_packer)
         except ValueError as e:
             self.assertIn("could not serialize", str(e))
             self.assertIn("don't work", str(e))
         else:
             self.fail("Should have raised ValueError")
     
     def test_bad_unpack(self):
         try:
             session = ss.Session(unpack=_bad_unpacker)
         except ValueError as e:
             self.assertIn("could not handle output", str(e))
             self.assertIn("don't work either", str(e))
         else:
             self.fail("Should have raised ValueError")
     
     def test_bad_packer(self):
         try:
             session = ss.Session(packer=__name__ + '._bad_packer')
         except ValueError as e:
             self.assertIn("could not serialize", str(e))
             self.assertIn("don't work", str(e))
         else:
             self.fail("Should have raised ValueError")
     
     def test_bad_unpacker(self):
         try:
             session = ss.Session(unpacker=__name__ + '._bad_unpacker')
         except ValueError as e:
             self.assertIn("could not handle output", str(e))
             self.assertIn("don't work either", str(e))
         else:
             self.fail("Should have raised ValueError")
     
     def test_bad_roundtrip(self):
         with self.assertRaises(ValueError):
             session = ss.Session(unpack=lambda b: 5)
     
     def _datetime_test(self, session):
         content = dict(t=datetime.now())
         metadata = dict(t=datetime.now())
         p = session.msg('msg')
         msg = session.msg('msg', content=content, metadata=metadata, parent=p['header'])
         smsg = session.serialize(msg)
         msg2 = session.deserialize(session.feed_identities(smsg)[1])
         assert isinstance(msg2['header']['date'], datetime)
         self.assertEqual(msg['header'], msg2['header'])
         self.assertEqual(msg['parent_header'], msg2['parent_header'])
         self.assertEqual(msg['parent_header'], msg2['parent_header'])
         assert isinstance(msg['content']['t'], datetime)
         assert isinstance(msg['metadata']['t'], datetime)
         assert isinstance(msg2['content']['t'], string_types)
         assert isinstance(msg2['metadata']['t'], string_types)
         self.assertEqual(msg['content'], jsonutil.extract_dates(msg2['content']))
         self.assertEqual(msg['content'], jsonutil.extract_dates(msg2['content']))
     
     def test_datetimes(self):
         self._datetime_test(self.session)
     
     def test_datetimes_pickle(self):
         session = ss.Session(packer='pickle')
         self._datetime_test(session)
     
     @skipif(module_not_available('msgpack'))
     def test_datetimes_msgpack(self):
         import msgpack
         
         session = ss.Session(
             pack=msgpack.packb,
             unpack=lambda buf: msgpack.unpackb(buf, encoding='utf8'),
         )
         self._datetime_test(session)
     
     def test_send_raw(self):
         ctx = zmq.Context.instance()
         A = ctx.socket(zmq.PAIR)
         B = ctx.socket(zmq.PAIR)
         A.bind("inproc://test")
         B.connect("inproc://test")
 
         msg = self.session.msg('execute', content=dict(a=10))
         msg_list = [self.session.pack(msg[part]) for part in 
                     ['header', 'parent_header', 'metadata', 'content']]
         self.session.send_raw(A, msg_list, ident=b'foo')
         
         ident, new_msg_list = self.session.feed_identities(B.recv_multipart())
         new_msg = self.session.deserialize(new_msg_list)
         self.assertEqual(ident[0], b'foo')
         self.assertEqual(new_msg['msg_type'],msg['msg_type'])
         self.assertEqual(new_msg['header'],msg['header'])
         self.assertEqual(new_msg['parent_header'],msg['parent_header'])
         self.assertEqual(new_msg['content'],msg['content'])
         self.assertEqual(new_msg['metadata'],msg['metadata'])
 
         A.close()
         B.close()
         ctx.term()

IPython/parallel/apps/ipcontrollerapp.py

@@ -1,547 +1,545 b''
 #!/usr/bin/env python
 # encoding: utf-8
 """
 The IPython controller application.
 
 Authors:
 
 * Brian Granger
 * MinRK
 
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import with_statement
 
 import json
 import os
 import stat
 import sys
 
 from multiprocessing import Process
 from signal import signal, SIGINT, SIGABRT, SIGTERM
 
 import zmq
 from zmq.devices import ProcessMonitoredQueue
 from zmq.log.handlers import PUBHandler
 
 from IPython.core.profiledir import ProfileDir
 
 from IPython.parallel.apps.baseapp import (
     BaseParallelApplication,
     base_aliases,
     base_flags,
     catch_config_error,
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.localinterfaces import localhost, public_ips
 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
 
 from IPython.kernel.zmq.session import (
-    Session, session_aliases, session_flags, default_secure
+    Session, session_aliases, session_flags,
 )
 
 from IPython.parallel.controller.heartmonitor import HeartMonitor
 from IPython.parallel.controller.hub import HubFactory
 from IPython.parallel.controller.scheduler import TaskScheduler,launch_scheduler
 from IPython.parallel.controller.dictdb import DictDB
 
 from IPython.parallel.util import split_url, disambiguate_url, set_hwm
 
 # conditional import of SQLiteDB / MongoDB backend class
 real_dbs = []
 
 try:
     from IPython.parallel.controller.sqlitedb import SQLiteDB
 except ImportError:
     pass
 else:
     real_dbs.append(SQLiteDB)
 
 try:
     from IPython.parallel.controller.mongodb import MongoDB
 except ImportError:
     pass
 else:
     real_dbs.append(MongoDB)
 
 
 
 #-----------------------------------------------------------------------------
 # Module level variables
 #-----------------------------------------------------------------------------
 
 
 _description = """Start the IPython controller for parallel computing.
 
 The IPython controller provides a gateway between the IPython engines and
 clients. The controller needs to be started before the engines and can be
 configured using command line options or using a cluster directory. Cluster
 directories contain config, log and security files and are usually located in
 your ipython directory and named as "profile_name". See the `profile`
 and `profile-dir` options for details.
 """
 
 _examples = """
 ipcontroller --ip=192.168.0.1 --port=1000  # listen on ip, port for engines
 ipcontroller --scheme=pure  # use the pure zeromq scheduler
 """
 
 
 #-----------------------------------------------------------------------------
 # The main application
 #-----------------------------------------------------------------------------
 flags = {}
 flags.update(base_flags)
 flags.update({
     'usethreads' : ( {'IPControllerApp' : {'use_threads' : True}},
                     'Use threads instead of processes for the schedulers'),
     'sqlitedb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.sqlitedb.SQLiteDB'}},
                     'use the SQLiteDB backend'),
     'mongodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.mongodb.MongoDB'}},
                     'use the MongoDB backend'),
     'dictdb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.DictDB'}},
                     'use the in-memory DictDB backend'),
     'nodb' : ({'HubFactory' : {'db_class' : 'IPython.parallel.controller.dictdb.NoDB'}},
                     """use dummy DB backend, which doesn't store any information.
                     
                     This is the default as of IPython 0.13.
                     
                     To enable delayed or repeated retrieval of results from the Hub,
                     select one of the true db backends.
                     """),
     'reuse' : ({'IPControllerApp' : {'reuse_files' : True}},
                     'reuse existing json connection files'),
     'restore' : ({'IPControllerApp' : {'restore_engines' : True, 'reuse_files' : True}},
                     'Attempt to restore engines from a JSON file.  '
                     'For use when resuming a crashed controller'),
 })
 
 flags.update(session_flags)
 
 aliases = dict(
     ssh = 'IPControllerApp.ssh_server',
     enginessh = 'IPControllerApp.engine_ssh_server',
     location = 'IPControllerApp.location',
 
     url = 'HubFactory.url',
     ip = 'HubFactory.ip',
     transport = 'HubFactory.transport',
     port = 'HubFactory.regport',
 
     ping = 'HeartMonitor.period',
 
     scheme = 'TaskScheduler.scheme_name',
     hwm = 'TaskScheduler.hwm',
 )
 aliases.update(base_aliases)
 aliases.update(session_aliases)
 
 class IPControllerApp(BaseParallelApplication):
 
     name = u'ipcontroller'
     description = _description
     examples = _examples
     classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB] + real_dbs
     
     # change default to True
     auto_create = Bool(True, config=True,
         help="""Whether to create profile dir if it doesn't exist.""")
     
     reuse_files = Bool(False, config=True,
         help="""Whether to reuse existing json connection files.
         If False, connection files will be removed on a clean exit.
         """
     )
     restore_engines = Bool(False, config=True,
         help="""Reload engine state from JSON file
         """
     )
     ssh_server = Unicode(u'', config=True,
         help="""ssh url for clients to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     engine_ssh_server = Unicode(u'', config=True,
         help="""ssh url for engines to use when connecting to the Controller
         processes. It should be of the form: [user@]server[:port]. The
         Controller's listening addresses must be accessible from the ssh server""",
     )
     location = Unicode(u'', config=True,
         help="""The external IP or domain name of the Controller, used for disambiguating
         engine and client connections.""",
     )
     import_statements = List([], config=True,
         help="import statements to be run at startup.  Necessary in some environments"
     )
 
     use_threads = Bool(False, config=True,
         help='Use threads instead of processes for the schedulers',
     )
 
     engine_json_file = Unicode('ipcontroller-engine.json', config=True,
         help="JSON filename where engine connection info will be stored.")
     client_json_file = Unicode('ipcontroller-client.json', config=True,
         help="JSON filename where client connection info will be stored.")
     
     def _cluster_id_changed(self, name, old, new):
         super(IPControllerApp, self)._cluster_id_changed(name, old, new)
         self.engine_json_file = "%s-engine.json" % self.name
         self.client_json_file = "%s-client.json" % self.name
 
 
     # internal
     children = List()
     mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')
 
     def _use_threads_changed(self, name, old, new):
         self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
     
     write_connection_files = Bool(True,
         help="""Whether to write connection files to disk.
         True in all cases other than runs with `reuse_files=True` *after the first*
         """
     )
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     
 
     def save_connection_dict(self, fname, cdict):
         """save a connection dict to json file."""
         c = self.config
         url = cdict['registration']
         location = cdict['location']
         
         if not location:
             if public_ips():
                 location = public_ips()[-1]
             else:
                 self.log.warn("Could not identify this machine's IP, assuming %s."
                 " You may need to specify '--location=<external_ip_address>' to help"
                 " IPython decide when to connect via loopback." % localhost() )
                 location = localhost()
             cdict['location'] = location
         fname = os.path.join(self.profile_dir.security_dir, fname)
         self.log.info("writing connection info to %s", fname)
         with open(fname, 'w') as f:
             f.write(json.dumps(cdict, indent=2))
         os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
     
     def load_config_from_json(self):
         """load config from existing json connector files."""
         c = self.config
         self.log.debug("loading config from JSON")
         
         # load engine config
         
         fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
         self.log.info("loading connection info from %s", fname)
         with open(fname) as f:
             ecfg = json.loads(f.read())
         
         # json gives unicode, Session.key wants bytes
         c.Session.key = ecfg['key'].encode('ascii')
         
         xport,ip = ecfg['interface'].split('://')
         
         c.HubFactory.engine_ip = ip
         c.HubFactory.engine_transport = xport
         
         self.location = ecfg['location']
         if not self.engine_ssh_server:
             self.engine_ssh_server = ecfg['ssh']
         
         # load client config
         
         fname = os.path.join(self.profile_dir.security_dir, self.client_json_file)
         self.log.info("loading connection info from %s", fname)
         with open(fname) as f:
             ccfg = json.loads(f.read())
         
         for key in ('key', 'registration', 'pack', 'unpack', 'signature_scheme'):
             assert ccfg[key] == ecfg[key], "mismatch between engine and client info: %r" % key
         
         xport,addr = ccfg['interface'].split('://')
         
         c.HubFactory.client_transport = xport
         c.HubFactory.client_ip = ip
         if not self.ssh_server:
             self.ssh_server = ccfg['ssh']
         
         # load port config:
         c.HubFactory.regport = ecfg['registration']
         c.HubFactory.hb = (ecfg['hb_ping'], ecfg['hb_pong'])
         c.HubFactory.control = (ccfg['control'], ecfg['control'])
         c.HubFactory.mux = (ccfg['mux'], ecfg['mux'])
         c.HubFactory.task = (ccfg['task'], ecfg['task'])
         c.HubFactory.iopub = (ccfg['iopub'], ecfg['iopub'])
         c.HubFactory.notifier_port = ccfg['notification']
     
     def cleanup_connection_files(self):
         if self.reuse_files:
             self.log.debug("leaving JSON connection files for reuse")
             return
         self.log.debug("cleaning up JSON connection files")
         for f in (self.client_json_file, self.engine_json_file):
             f = os.path.join(self.profile_dir.security_dir, f)
             try:
                 os.remove(f)
             except Exception as e:
                 self.log.error("Failed to cleanup connection file: %s", e)
             else:
                 self.log.debug(u"removed %s", f)
     
     def load_secondary_config(self):
         """secondary config, loading from JSON and setting defaults"""
         if self.reuse_files:
             try:
                 self.load_config_from_json()
             except (AssertionError,IOError) as e:
                 self.log.error("Could not load config from JSON: %s" % e)
             else:
                 # successfully loaded config from JSON, and reuse=True
                 # no need to wite back the same file
                 self.write_connection_files = False
                 
-        # switch Session.key default to secure
-        default_secure(self.config)
         self.log.debug("Config changed")
         self.log.debug(repr(self.config))
         
     def init_hub(self):
         c = self.config
         
         self.do_import_statements()
         
         try:
             self.factory = HubFactory(config=c, log=self.log)
             # self.start_logging()
             self.factory.init_hub()
         except TraitError:
             raise
         except Exception:
             self.log.error("Couldn't construct the Controller", exc_info=True)
             self.exit(1)
         
         if self.write_connection_files:
             # save to new json config files
             f = self.factory
             base = {
                 'key'  : f.session.key.decode('ascii'),
                 'location'  : self.location,
                 'pack'      : f.session.packer,
                 'unpack'    : f.session.unpacker,
                 'signature_scheme' : f.session.signature_scheme,
             }
             
             cdict = {'ssh' : self.ssh_server}
             cdict.update(f.client_info)
             cdict.update(base)
             self.save_connection_dict(self.client_json_file, cdict)
             
             edict = {'ssh' : self.engine_ssh_server}
             edict.update(f.engine_info)
             edict.update(base)
             self.save_connection_dict(self.engine_json_file, edict)
 
         fname = "engines%s.json" % self.cluster_id
         self.factory.hub.engine_state_file = os.path.join(self.profile_dir.log_dir, fname)
         if self.restore_engines:
             self.factory.hub._load_engine_state()
 
     def init_schedulers(self):
         children = self.children
         mq = import_item(str(self.mq_class))
         
         f = self.factory
         ident = f.session.bsession
         # disambiguate url, in case of *
         monitor_url = disambiguate_url(f.monitor_url)
         # maybe_inproc = 'inproc://monitor' if self.use_threads else monitor_url
         # IOPub relay (in a Process)
         q = mq(zmq.PUB, zmq.SUB, zmq.PUB, b'N/A',b'iopub')
         q.bind_in(f.client_url('iopub'))
         q.setsockopt_in(zmq.IDENTITY, ident + b"_iopub")
         q.bind_out(f.engine_url('iopub'))
         q.setsockopt_out(zmq.SUBSCRIBE, b'')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
 
         # Multiplexer Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'in', b'out')
         
         q.bind_in(f.client_url('mux'))
         q.setsockopt_in(zmq.IDENTITY, b'mux_in')
         q.bind_out(f.engine_url('mux'))
         q.setsockopt_out(zmq.IDENTITY, b'mux_out')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
 
         # Control Queue (in a Process)
         q = mq(zmq.ROUTER, zmq.ROUTER, zmq.PUB, b'incontrol', b'outcontrol')
         q.bind_in(f.client_url('control'))
         q.setsockopt_in(zmq.IDENTITY, b'control_in')
         q.bind_out(f.engine_url('control'))
         q.setsockopt_out(zmq.IDENTITY, b'control_out')
         q.connect_mon(monitor_url)
         q.daemon=True
         children.append(q)
         if 'TaskScheduler.scheme_name' in self.config:
             scheme = self.config.TaskScheduler.scheme_name
         else:
             scheme = TaskScheduler.scheme_name.get_default_value()
         # Task Queue (in a Process)
         if scheme == 'pure':
             self.log.warn("task::using pure DEALER Task scheduler")
             q = mq(zmq.ROUTER, zmq.DEALER, zmq.PUB, b'intask', b'outtask')
             # q.setsockopt_out(zmq.HWM, hub.hwm)
             q.bind_in(f.client_url('task'))
             q.setsockopt_in(zmq.IDENTITY, b'task_in')
             q.bind_out(f.engine_url('task'))
             q.setsockopt_out(zmq.IDENTITY, b'task_out')
             q.connect_mon(monitor_url)
             q.daemon=True
             children.append(q)
         elif scheme == 'none':
             self.log.warn("task::using no Task scheduler")
 
         else:
             self.log.info("task::using Python %s Task scheduler"%scheme)
             sargs = (f.client_url('task'), f.engine_url('task'),
                     monitor_url, disambiguate_url(f.client_url('notification')),
                     disambiguate_url(f.client_url('registration')),
             )
             kwargs = dict(logname='scheduler', loglevel=self.log_level,
                             log_url = self.log_url, config=dict(self.config))
             if 'Process' in self.mq_class:
                 # run the Python scheduler in a Process
                 q = Process(target=launch_scheduler, args=sargs, kwargs=kwargs)
                 q.daemon=True
                 children.append(q)
             else:
                 # single-threaded Controller
                 kwargs['in_thread'] = True
                 launch_scheduler(*sargs, **kwargs)
         
         # set unlimited HWM for all relay devices
         if hasattr(zmq, 'SNDHWM'):
             q = children[0]
             q.setsockopt_in(zmq.RCVHWM, 0)
             q.setsockopt_out(zmq.SNDHWM, 0)
             
             for q in children[1:]:
                 if not hasattr(q, 'setsockopt_in'):
                     continue
                 q.setsockopt_in(zmq.SNDHWM, 0)
                 q.setsockopt_in(zmq.RCVHWM, 0)
                 q.setsockopt_out(zmq.SNDHWM, 0)
                 q.setsockopt_out(zmq.RCVHWM, 0)
                 q.setsockopt_mon(zmq.SNDHWM, 0)
             
 
     def terminate_children(self):
         child_procs = []
         for child in self.children:
             if isinstance(child, ProcessMonitoredQueue):
                 child_procs.append(child.launcher)
             elif isinstance(child, Process):
                 child_procs.append(child)
         if child_procs:
             self.log.critical("terminating children...")
             for child in child_procs:
                 try:
                     child.terminate()
                 except OSError:
                     # already dead
                     pass
     
     def handle_signal(self, sig, frame):
         self.log.critical("Received signal %i, shutting down", sig)
         self.terminate_children()
         self.loop.stop()
     
     def init_signal(self):
         for sig in (SIGINT, SIGABRT, SIGTERM):
             signal(sig, self.handle_signal)
 
     def do_import_statements(self):
         statements = self.import_statements
         for s in statements:
             try:
                 self.log.msg("Executing statement: '%s'" % s)
                 exec(s, globals(), locals())
             except:
                 self.log.msg("Error running statement: %s" % s)
 
     def forward_logging(self):
         if self.log_url:
             self.log.info("Forwarding logging to %s"%self.log_url)
             context = zmq.Context.instance()
             lsock = context.socket(zmq.PUB)
             lsock.connect(self.log_url)
             handler = PUBHandler(lsock)
             handler.root_topic = 'controller'
             handler.setLevel(self.log_level)
             self.log.addHandler(handler)
     
     @catch_config_error
     def initialize(self, argv=None):
         super(IPControllerApp, self).initialize(argv)
         self.forward_logging()
         self.load_secondary_config()
         self.init_hub()
         self.init_schedulers()
     
     def start(self):
         # Start the subprocesses:
         self.factory.start()
         # children must be started before signals are setup,
         # otherwise signal-handling will fire multiple times
         for child in self.children:
             child.start()
         self.init_signal()
 
         self.write_pid_file(overwrite=True)
 
         try:
             self.factory.loop.start()
         except KeyboardInterrupt:
             self.log.critical("Interrupted, Exiting...\n")
         finally:
             self.cleanup_connection_files()
             
 
 def launch_new_instance(*args, **kwargs):
     """Create and run the IPython controller"""
     if sys.platform == 'win32':
         # make sure we don't get called from a multiprocessing subprocess
         # this can result in infinite Controllers being started on Windows
         # which doesn't have a proper fork, so multiprocessing is wonky
         
         # this only comes up when IPython has been installed using vanilla
         # setuptools, and *not* distribute.
         import multiprocessing
         p = multiprocessing.current_process()
         # the main process has name 'MainProcess'
         # subprocesses will have names like 'Process-1'
         if p.name != 'MainProcess':
             # we are a subprocess, don't start another Controller!
             return
     return IPControllerApp.launch_instance(*args, **kwargs)
 
 
 if __name__ == '__main__':
     launch_new_instance()