Merge remote-tracking branch 'upstream/master'

IPython/html/services/security/__init__.py

@@ -0,0 +1,4 b''
+# URI for the CSP Report. Included here to prevent a cyclic dependency.
+# csp_report_uri is needed both by the BaseHandler (for setting the report-uri)
+# and by the CSPReportHandler (which depends on the BaseHandler).
+csp_report_uri = r"/api/security/csp-report"

IPython/html/services/security/handlers.py

@@ -0,0 +1,23 b''
+"""Tornado handlers for security logging."""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+from tornado import gen, web
+
+from ...base.handlers import IPythonHandler, json_errors
+from . import csp_report_uri
+
+class CSPReportHandler(IPythonHandler):
+    '''Accepts a content security policy violation report'''
+    @web.authenticated
+    @json_errors
+    def post(self):
+        '''Log a content security policy violation report'''
+        csp_report = self.get_json_body()
+        self.log.warn("Content security violation: %s",
+                      self.request.body.decode('utf8', 'replace'))
+
+default_handlers = [
+    (csp_report_uri, CSPReportHandler)
+]

IPython/config/application.py

@@ -1,601 +1,611 b''
 # encoding: utf-8
 """A base class for a configurable application."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
+import json
 import logging
 import os
 import re
 import sys
 from copy import deepcopy
 from collections import defaultdict
 
 from IPython.external.decorator import decorator
 
 from IPython.config.configurable import SingletonConfigurable
 from IPython.config.loader import (
     KVArgParseConfigLoader, PyFileConfigLoader, Config, ArgumentError, ConfigFileNotFound, JSONFileConfigLoader
 )
 
 from IPython.utils.traitlets import (
     Unicode, List, Enum, Dict, Instance, TraitError
 )
 from IPython.utils.importstring import import_item
 from IPython.utils.text import indent, wrap_paragraphs, dedent
 from IPython.utils import py3compat
 from IPython.utils.py3compat import string_types, iteritems
 
 #-----------------------------------------------------------------------------
 # Descriptions for the various sections
 #-----------------------------------------------------------------------------
 
 # merge flags&aliases into options
 option_description = """
 Arguments that take values are actually convenience aliases to full
 Configurables, whose aliases are listed on the help line. For more information
 on full configurables, see '--help-all'.
 """.strip() # trim newlines of front and back
 
 keyvalue_description = """
 Parameters are set from command-line arguments of the form:
 `--Class.trait=value`.
 This line is evaluated in Python, so simple expressions are allowed, e.g.::
 `--C.a='range(3)'` For setting C.a=[0,1,2].
 """.strip() # trim newlines of front and back
 
 # sys.argv can be missing, for example when python is embedded. See the docs
 # for details: http://docs.python.org/2/c-api/intro.html#embedding-python
 if not hasattr(sys, "argv"):
     sys.argv = [""]
 
 subcommand_description = """
 Subcommands are launched as `{app} cmd [args]`. For information on using
 subcommand 'cmd', do: `{app} cmd -h`.
 """
 # get running program name
 
 #-----------------------------------------------------------------------------
 # Application class
 #-----------------------------------------------------------------------------
 
 @decorator
 def catch_config_error(method, app, *args, **kwargs):
     """Method decorator for catching invalid config (Trait/ArgumentErrors) during init.
 
     On a TraitError (generally caused by bad config), this will print the trait's
     message, and exit the app.
     
     For use on init methods, to prevent invoking excepthook on invalid input.
     """
     try:
         return method(app, *args, **kwargs)
     except (TraitError, ArgumentError) as e:
         app.print_help()
         app.log.fatal("Bad config encountered during initialization:")
         app.log.fatal(str(e))
         app.log.debug("Config at the time: %s", app.config)
         app.exit(1)
 
 
 class ApplicationError(Exception):
     pass
 
 class LevelFormatter(logging.Formatter):
     """Formatter with additional `highlevel` record
     
     This field is empty if log level is less than highlevel_limit,
     otherwise it is formatted with self.highlevel_format.
     
     Useful for adding 'WARNING' to warning messages,
     without adding 'INFO' to info, etc.
     """
     highlevel_limit = logging.WARN
     highlevel_format = " %(levelname)s |"
     
     def format(self, record):
         if record.levelno >= self.highlevel_limit:
             record.highlevel = self.highlevel_format % record.__dict__
         else:
             record.highlevel = ""
         return super(LevelFormatter, self).format(record)
             
 
 class Application(SingletonConfigurable):
     """A singleton application with full configuration support."""
 
     # The name of the application, will usually match the name of the command
     # line application
     name = Unicode(u'application')
 
     # The description of the application that is printed at the beginning
     # of the help.
     description = Unicode(u'This is an application.')
     # default section descriptions
     option_description = Unicode(option_description)
     keyvalue_description = Unicode(keyvalue_description)
     subcommand_description = Unicode(subcommand_description)
 
     # The usage and example string that goes at the end of the help string.
     examples = Unicode()
 
     # A sequence of Configurable subclasses whose config=True attributes will
     # be exposed at the command line.
     classes = []
     @property
     def _help_classes(self):
         """Define `App.help_classes` if CLI classes should differ from config file classes"""
         return getattr(self, 'help_classes', self.classes)
     
     @property
     def _config_classes(self):
         """Define `App.config_classes` if config file classes should differ from CLI classes."""
         return getattr(self, 'config_classes', self.classes)
 
     # The version string of this application.
     version = Unicode(u'0.0')
     
     # the argv used to initialize the application
     argv = List()
 
     # The log level for the application
     log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
                     default_value=logging.WARN,
                     config=True,
                     help="Set the log level by value or name.")
     def _log_level_changed(self, name, old, new):
         """Adjust the log level when log_level is set."""
         if isinstance(new, string_types):
             new = getattr(logging, new)
             self.log_level = new
         self.log.setLevel(new)
     
     _log_formatter_cls = LevelFormatter
     
     log_datefmt = Unicode("%Y-%m-%d %H:%M:%S", config=True,
         help="The date format used by logging formatters for %(asctime)s"
     )
     def _log_datefmt_changed(self, name, old, new):
         self._log_format_changed()
     
     log_format = Unicode("[%(name)s]%(highlevel)s %(message)s", config=True,
         help="The Logging format template",
     )
     def _log_format_changed(self, name, old, new):
         """Change the log formatter when log_format is set."""
         _log_handler = self.log.handlers[0]
         _log_formatter = self._log_formatter_cls(fmt=new, datefmt=self.log_datefmt)
         _log_handler.setFormatter(_log_formatter)
     
 
     log = Instance(logging.Logger)
     def _log_default(self):
         """Start logging for this application.
 
         The default is to log to stderr using a StreamHandler, if no default
         handler already exists.  The log level starts at logging.WARN, but this
         can be adjusted by setting the ``log_level`` attribute.
         """
         log = logging.getLogger(self.__class__.__name__)
         log.setLevel(self.log_level)
         log.propagate = False
         _log = log # copied from Logger.hasHandlers() (new in Python 3.2)
         while _log:
             if _log.handlers:
                 return log
             if not _log.propagate:
                 break
             else:
                 _log = _log.parent
         if sys.executable.endswith('pythonw.exe'):
             # this should really go to a file, but file-logging is only
             # hooked up in parallel applications
             _log_handler = logging.StreamHandler(open(os.devnull, 'w'))
         else:
             _log_handler = logging.StreamHandler()
         _log_formatter = self._log_formatter_cls(fmt=self.log_format, datefmt=self.log_datefmt)
         _log_handler.setFormatter(_log_formatter)
         log.addHandler(_log_handler)
         return log
 
     # the alias map for configurables
     aliases = Dict({'log-level' : 'Application.log_level'})
 
     # flags for loading Configurables or store_const style flags
     # flags are loaded from this dict by '--key' flags
     # this must be a dict of two-tuples, the first element being the Config/dict
     # and the second being the help string for the flag
     flags = Dict()
     def _flags_changed(self, name, old, new):
         """ensure flags dict is valid"""
         for key,value in iteritems(new):
             assert len(value) == 2, "Bad flag: %r:%s"%(key,value)
             assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value)
             assert isinstance(value[1], string_types), "Bad flag: %r:%s"%(key,value)
 
 
     # subcommands for launching other applications
     # if this is not empty, this will be a parent Application
     # this must be a dict of two-tuples,
     # the first element being the application class/import string
     # and the second being the help string for the subcommand
     subcommands = Dict()
     # parse_command_line will initialize a subapp, if requested
     subapp = Instance('IPython.config.application.Application', allow_none=True)
 
     # extra command-line arguments that don't set config values
     extra_args = List(Unicode)
 
 
     def __init__(self, **kwargs):
         SingletonConfigurable.__init__(self, **kwargs)
         # Ensure my class is in self.classes, so my attributes appear in command line
         # options and config files.
         if self.__class__ not in self.classes:
             self.classes.insert(0, self.__class__)
 
     def _config_changed(self, name, old, new):
         SingletonConfigurable._config_changed(self, name, old, new)
         self.log.debug('Config changed:')
         self.log.debug(repr(new))
 
     @catch_config_error
     def initialize(self, argv=None):
         """Do the basic steps to configure me.
 
         Override in subclasses.
         """
         self.parse_command_line(argv)
 
 
     def start(self):
         """Start the app mainloop.
 
         Override in subclasses.
         """
         if self.subapp is not None:
             return self.subapp.start()
 
     def print_alias_help(self):
         """Print the alias part of the help."""
         if not self.aliases:
             return
 
         lines = []
         classdict = {}
         for cls in self._help_classes:
             # include all parents (up to, but excluding Configurable) in available names
             for c in cls.mro()[:-3]:
                 classdict[c.__name__] = c
 
         for alias, longname in iteritems(self.aliases):
             classname, traitname = longname.split('.',1)
             cls = classdict[classname]
 
             trait = cls.class_traits(config=True)[traitname]
             help = cls.class_get_trait_help(trait).splitlines()
             # reformat first line
             help[0] = help[0].replace(longname, alias) + ' (%s)'%longname
             if len(alias) == 1:
                 help[0] = help[0].replace('--%s='%alias, '-%s '%alias)
             lines.extend(help)
         # lines.append('')
         print(os.linesep.join(lines))
 
     def print_flag_help(self):
         """Print the flag part of the help."""
         if not self.flags:
             return
 
         lines = []
         for m, (cfg,help) in iteritems(self.flags):
             prefix = '--' if len(m) > 1 else '-'
             lines.append(prefix+m)
             lines.append(indent(dedent(help.strip())))
         # lines.append('')
         print(os.linesep.join(lines))
 
     def print_options(self):
         if not self.flags and not self.aliases:
             return
         lines = ['Options']
         lines.append('-'*len(lines[0]))
         lines.append('')
         for p in wrap_paragraphs(self.option_description):
             lines.append(p)
             lines.append('')
         print(os.linesep.join(lines))
         self.print_flag_help()
         self.print_alias_help()
         print()
 
     def print_subcommands(self):
         """Print the subcommand part of the help."""
         if not self.subcommands:
             return
 
         lines = ["Subcommands"]
         lines.append('-'*len(lines[0]))
         lines.append('')
         for p in wrap_paragraphs(self.subcommand_description.format(
                     app=self.name)):
             lines.append(p)
             lines.append('')
         for subc, (cls, help) in iteritems(self.subcommands):
             lines.append(subc)
             if help:
                 lines.append(indent(dedent(help.strip())))
         lines.append('')
         print(os.linesep.join(lines))
 
     def print_help(self, classes=False):
         """Print the help for each Configurable class in self.classes.
 
         If classes=False (the default), only flags and aliases are printed.
         """
         self.print_description()
         self.print_subcommands()
         self.print_options()
 
         if classes:
             help_classes = self._help_classes
             if help_classes:
                 print("Class parameters")
                 print("----------------")
                 print()
                 for p in wrap_paragraphs(self.keyvalue_description):
                     print(p)
                     print()
 
             for cls in help_classes:
                 cls.class_print_help()
                 print()
         else:
             print("To see all available configurables, use `--help-all`")
             print()
 
         self.print_examples()
 
 
     def print_description(self):
         """Print the application description."""
         for p in wrap_paragraphs(self.description):
             print(p)
             print()
 
     def print_examples(self):
         """Print usage and examples.
 
         This usage string goes at the end of the command line help string
         and should contain examples of the application's usage.
         """
         if self.examples:
             print("Examples")
             print("--------")
             print()
             print(indent(dedent(self.examples.strip())))
             print()
 
     def print_version(self):
         """Print the version string."""
         print(self.version)
 
     def update_config(self, config):
         """Fire the traits events when the config is updated."""
         # Save a copy of the current config.
         newconfig = deepcopy(self.config)
         # Merge the new config into the current one.
         newconfig.merge(config)
         # Save the combined config as self.config, which triggers the traits
         # events.
         self.config = newconfig
 
     @catch_config_error
     def initialize_subcommand(self, subc, argv=None):
         """Initialize a subcommand with argv."""
         subapp,help = self.subcommands.get(subc)
 
         if isinstance(subapp, string_types):
             subapp = import_item(subapp)
 
         # clear existing instances
         self.__class__.clear_instance()
         # instantiate
         self.subapp = subapp.instance(config=self.config)
         # and initialize subapp
         self.subapp.initialize(argv)
     
     def flatten_flags(self):
         """flatten flags and aliases, so cl-args override as expected.
         
         This prevents issues such as an alias pointing to InteractiveShell,
         but a config file setting the same trait in TerminalInteraciveShell
         getting inappropriate priority over the command-line arg.
 
         Only aliases with exactly one descendent in the class list
         will be promoted.
         
         """
         # build a tree of classes in our list that inherit from a particular
         # it will be a dict by parent classname of classes in our list
         # that are descendents
         mro_tree = defaultdict(list)
         for cls in self._help_classes:
             clsname = cls.__name__
             for parent in cls.mro()[1:-3]:
                 # exclude cls itself and Configurable,HasTraits,object
                 mro_tree[parent.__name__].append(clsname)
         # flatten aliases, which have the form:
         # { 'alias' : 'Class.trait' }
         aliases = {}
         for alias, cls_trait in iteritems(self.aliases):
             cls,trait = cls_trait.split('.',1)
             children = mro_tree[cls]
             if len(children) == 1:
                 # exactly one descendent, promote alias
                 cls = children[0]
             aliases[alias] = '.'.join([cls,trait])
         
         # flatten flags, which are of the form:
         # { 'key' : ({'Cls' : {'trait' : value}}, 'help')}
         flags = {}
         for key, (flagdict, help) in iteritems(self.flags):
             newflag = {}
             for cls, subdict in iteritems(flagdict):
                 children = mro_tree[cls]
                 # exactly one descendent, promote flag section
                 if len(children) == 1:
                     cls = children[0]
                 newflag[cls] = subdict
             flags[key] = (newflag, help)
         return flags, aliases
 
     @catch_config_error
     def parse_command_line(self, argv=None):
         """Parse the command line arguments."""
         argv = sys.argv[1:] if argv is None else argv
         self.argv = [ py3compat.cast_unicode(arg) for arg in argv ]
         
         if argv and argv[0] == 'help':
             # turn `ipython help notebook` into `ipython notebook -h`
             argv = argv[1:] + ['-h']
 
         if self.subcommands and len(argv) > 0:
             # we have subcommands, and one may have been specified
             subc, subargv = argv[0], argv[1:]
             if re.match(r'^\w(\-?\w)*$', subc) and subc in self.subcommands:
                 # it's a subcommand, and *not* a flag or class parameter
                 return self.initialize_subcommand(subc, subargv)
 
         # Arguments after a '--' argument are for the script IPython may be
         # about to run, not IPython iteslf. For arguments parsed here (help and
         # version), we want to only search the arguments up to the first
         # occurrence of '--', which we're calling interpreted_argv.
         try:
             interpreted_argv = argv[:argv.index('--')]
         except ValueError:
             interpreted_argv = argv
 
         if any(x in interpreted_argv for x in ('-h', '--help-all', '--help')):
             self.print_help('--help-all' in interpreted_argv)
             self.exit(0)
 
         if '--version' in interpreted_argv or '-V' in interpreted_argv:
             self.print_version()
             self.exit(0)
         
         # flatten flags&aliases, so cl-args get appropriate priority:
         flags,aliases = self.flatten_flags()
         loader = KVArgParseConfigLoader(argv=argv, aliases=aliases,
                                         flags=flags, log=self.log)
         config = loader.load_config()
         self.update_config(config)
         # store unparsed args in extra_args
         self.extra_args = loader.extra_args
 
     @classmethod
     def _load_config_files(cls, basefilename, path=None, log=None):
         """Load config files (py,json) by filename and path.
 
         yield each config object in turn.
         """
         
         if not isinstance(path, list):
             path = [path]
         for path in path[::-1]:
             # path list is in descending priority order, so load files backwards:
             pyloader = PyFileConfigLoader(basefilename+'.py', path=path, log=log)
             jsonloader = JSONFileConfigLoader(basefilename+'.json', path=path, log=log)
             config = None
             for loader in [pyloader, jsonloader]:
                 try:
                     config = loader.load_config()
                 except ConfigFileNotFound:
                     pass
                 except Exception:
                     # try to get the full filename, but it will be empty in the
                     # unlikely event that the error raised before filefind finished
                     filename = loader.full_filename or basefilename
                     # problem while running the file
                     if log:
                         log.error("Exception while loading config file %s",
                                 filename, exc_info=True)
                 else:
                     if log:
                         log.debug("Loaded config file: %s", loader.full_filename)
                 if config:
                      yield config
 
         raise StopIteration
 
 
     @catch_config_error
     def load_config_file(self, filename, path=None):
         """Load config files by filename and path."""
         filename, ext = os.path.splitext(filename)
+        loaded = []
         for config in self._load_config_files(filename, path=path, log=self.log):
+            loaded.append(config)
             self.update_config(config)
+        if len(loaded) > 1:
+            collisions = loaded[0].collisions(loaded[1])
+            if collisions:
+                self.log.warn("Collisions detected in {0}.py and {0}.json config files."
+                              " {0}.json has higher priority: {1}".format(
+                              filename, json.dumps(collisions, indent=2),
+                ))
 
 
     def generate_config_file(self):
         """generate default config file from Configurables"""
         lines = ["# Configuration file for %s."%self.name]
         lines.append('')
         lines.append('c = get_config()')
         lines.append('')
         for cls in self._config_classes:
             lines.append(cls.class_config_section())
         return '\n'.join(lines)
 
     def exit(self, exit_status=0):
         self.log.debug("Exiting application: %s" % self.name)
         sys.exit(exit_status)
 
     @classmethod
     def launch_instance(cls, argv=None, **kwargs):
         """Launch a global instance of this Application
         
         If a global instance already exists, this reinitializes and starts it
         """
         app = cls.instance(**kwargs)
         app.initialize(argv)
         app.start()
 
 #-----------------------------------------------------------------------------
 # utility functions, for convenience
 #-----------------------------------------------------------------------------
 
 def boolean_flag(name, configurable, set_help='', unset_help=''):
     """Helper for building basic --trait, --no-trait flags.
 
     Parameters
     ----------
 
     name : str
         The name of the flag.
     configurable : str
         The 'Class.trait' string of the trait to be set/unset with the flag
     set_help : unicode
         help string for --name flag
     unset_help : unicode
         help string for --no-name flag
 
     Returns
     -------
 
     cfg : dict
         A dict with two keys: 'name', and 'no-name', for setting and unsetting
         the trait, respectively.
     """
     # default helpstrings
     set_help = set_help or "set %s=True"%configurable
     unset_help = unset_help or "set %s=False"%configurable
 
     cls,trait = configurable.split('.')
 
     setter = {cls : {trait : True}}
     unsetter = {cls : {trait : False}}
     return {name : (setter, set_help), 'no-'+name : (unsetter, unset_help)}
 

IPython/config/loader.py

@@ -1,824 +1,844 b''
 # encoding: utf-8
 """A simple configuration system."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import argparse
 import copy
 import logging
 import os
 import re
 import sys
 import json
 
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 from IPython.utils.py3compat import unicode_type, iteritems
 from IPython.utils.traitlets import HasTraits, List, Any
 
 #-----------------------------------------------------------------------------
 # Exceptions
 #-----------------------------------------------------------------------------
 
 
 class ConfigError(Exception):
     pass
 
 class ConfigLoaderError(ConfigError):
     pass
 
 class ConfigFileNotFound(ConfigError):
     pass
 
 class ArgumentError(ConfigLoaderError):
     pass
 
 #-----------------------------------------------------------------------------
 # Argparse fix
 #-----------------------------------------------------------------------------
 
 # Unfortunately argparse by default prints help messages to stderr instead of
 # stdout.  This makes it annoying to capture long help screens at the command
 # line, since one must know how to pipe stderr, which many users don't know how
 # to do.  So we override the print_help method with one that defaults to
 # stdout and use our class instead.
 
 class ArgumentParser(argparse.ArgumentParser):
     """Simple argparse subclass that prints help to stdout by default."""
 
     def print_help(self, file=None):
         if file is None:
             file = sys.stdout
         return super(ArgumentParser, self).print_help(file)
 
     print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
 
 #-----------------------------------------------------------------------------
 # Config class for holding config information
 #-----------------------------------------------------------------------------
 
 class LazyConfigValue(HasTraits):
     """Proxy object for exposing methods on configurable containers
     
     Exposes:
     
     - append, extend, insert on lists
     - update on dicts
     - update, add on sets
     """
     
     _value = None
     
     # list methods
     _extend = List()
     _prepend = List()
     
     def append(self, obj):
         self._extend.append(obj)
     
     def extend(self, other):
         self._extend.extend(other)
     
     def prepend(self, other):
         """like list.extend, but for the front"""
         self._prepend[:0] = other
     
     _inserts = List()
     def insert(self, index, other):
         if not isinstance(index, int):
             raise TypeError("An integer is required")
         self._inserts.append((index, other))
     
     # dict methods
     # update is used for both dict and set
     _update = Any()
     def update(self, other):
         if self._update is None:
             if isinstance(other, dict):
                 self._update = {}
             else:
                 self._update = set()
         self._update.update(other)
     
     # set methods
     def add(self, obj):
         self.update({obj})
     
     def get_value(self, initial):
         """construct the value from the initial one
         
         after applying any insert / extend / update changes
         """
         if self._value is not None:
             return self._value
         value = copy.deepcopy(initial)
         if isinstance(value, list):
             for idx, obj in self._inserts:
                 value.insert(idx, obj)
             value[:0] = self._prepend
             value.extend(self._extend)
         
         elif isinstance(value, dict):
             if self._update:
                 value.update(self._update)
         elif isinstance(value, set):
             if self._update:
                 value.update(self._update)
         self._value = value
         return value
     
     def to_dict(self):
         """return JSONable dict form of my data
         
         Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
         """
         d = {}
         if self._update:
             d['update'] = self._update
         if self._extend:
             d['extend'] = self._extend
         if self._prepend:
             d['prepend'] = self._prepend
         elif self._inserts:
             d['inserts'] = self._inserts
         return d
 
 
 def _is_section_key(key):
     """Is a Config key a section name (does it start with a capital)?"""
     if key and key[0].upper()==key[0] and not key.startswith('_'):
         return True
     else:
         return False
 
 
 class Config(dict):
     """An attribute based dict that can do smart merges."""
 
     def __init__(self, *args, **kwds):
         dict.__init__(self, *args, **kwds)
         self._ensure_subconfig()
     
     def _ensure_subconfig(self):
         """ensure that sub-dicts that should be Config objects are
         
         casts dicts that are under section keys to Config objects,
         which is necessary for constructing Config objects from dict literals.
         """
         for key in self:
             obj = self[key]
             if _is_section_key(key) \
                     and isinstance(obj, dict) \
                     and not isinstance(obj, Config):
                 setattr(self, key, Config(obj))
     
     def _merge(self, other):
         """deprecated alias, use Config.merge()"""
         self.merge(other)
     
     def merge(self, other):
         """merge another config object into this one"""
         to_update = {}
         for k, v in iteritems(other):
             if k not in self:
                 to_update[k] = copy.deepcopy(v)
             else: # I have this key
                 if isinstance(v, Config) and isinstance(self[k], Config):
                     # Recursively merge common sub Configs
                     self[k].merge(v)
                 else:
                     # Plain updates for non-Configs
                     to_update[k] = copy.deepcopy(v)
 
         self.update(to_update)
 
+    def collisions(self, other):
+        """Check for collisions between two config objects.
+        
+        Returns a dict of the form {"Class": {"trait": "collision message"}}`,
+        indicating which values have been ignored.
+        
+        An empty dict indicates no collisions.
+        """
+        collisions = {}
+        for section in self:
+            if section not in other:
+                continue
+            mine = self[section]
+            theirs = other[section]
+            for key in mine:
+                if key in theirs and mine[key] != theirs[key]:
+                    collisions.setdefault(section, {})
+                    collisions[section][key] = "%r ignored, using %r" % (mine[key], theirs[key])
+        return collisions
+    
     def __contains__(self, key):
         # allow nested contains of the form `"Section.key" in config`
         if '.' in key:
             first, remainder = key.split('.', 1)
             if first not in self:
                 return False
             return remainder in self[first]
         
         return super(Config, self).__contains__(key)
     
     # .has_key is deprecated for dictionaries.
     has_key = __contains__
     
     def _has_section(self, key):
         return _is_section_key(key) and key in self
     
     def copy(self):
         return type(self)(dict.copy(self))
 
     def __copy__(self):
         return self.copy()
 
     def __deepcopy__(self, memo):
         import copy
         return type(self)(copy.deepcopy(list(self.items())))
     
     def __getitem__(self, key):
         try:
             return dict.__getitem__(self, key)
         except KeyError:
             if _is_section_key(key):
                 c = Config()
                 dict.__setitem__(self, key, c)
                 return c
             elif not key.startswith('_'):
                 # undefined, create lazy value, used for container methods
                 v = LazyConfigValue()
                 dict.__setitem__(self, key, v)
                 return v
             else:
                 raise KeyError
 
     def __setitem__(self, key, value):
         if _is_section_key(key):
             if not isinstance(value, Config):
                 raise ValueError('values whose keys begin with an uppercase '
                                  'char must be Config instances: %r, %r' % (key, value))
         dict.__setitem__(self, key, value)
 
     def __getattr__(self, key):
         if key.startswith('__'):
             return dict.__getattr__(self, key)
         try:
             return self.__getitem__(key)
         except KeyError as e:
             raise AttributeError(e)
 
     def __setattr__(self, key, value):
         if key.startswith('__'):
             return dict.__setattr__(self, key, value)
         try:
             self.__setitem__(key, value)
         except KeyError as e:
             raise AttributeError(e)
 
     def __delattr__(self, key):
         if key.startswith('__'):
             return dict.__delattr__(self, key)
         try:
             dict.__delitem__(self, key)
         except KeyError as e:
             raise AttributeError(e)
 
 
 #-----------------------------------------------------------------------------
 # Config loading classes
 #-----------------------------------------------------------------------------
 
 
 class ConfigLoader(object):
     """A object for loading configurations from just about anywhere.
 
     The resulting configuration is packaged as a :class:`Config`.
 
     Notes
     -----
     A :class:`ConfigLoader` does one thing: load a config from a source
     (file, command line arguments) and returns the data as a :class:`Config` object.
     There are lots of things that :class:`ConfigLoader` does not do.  It does
     not implement complex logic for finding config files.  It does not handle
     default values or merge multiple configs.  These things need to be
     handled elsewhere.
     """
 
     def _log_default(self):
         from IPython.utils.log import get_logger
         return get_logger()
 
     def __init__(self, log=None):
         """A base class for config loaders.
 
         log : instance of :class:`logging.Logger` to use.
               By default loger of :meth:`IPython.config.application.Application.instance()`
               will be used
 
         Examples
         --------
 
         >>> cl = ConfigLoader()
         >>> config = cl.load_config()
         >>> config
         {}
         """
         self.clear()
         if log is None:
             self.log = self._log_default()
             self.log.debug('Using default logger')
         else:
             self.log = log
 
     def clear(self):
         self.config = Config()
 
     def load_config(self):
         """Load a config from somewhere, return a :class:`Config` instance.
 
         Usually, this will cause self.config to be set and then returned.
         However, in most cases, :meth:`ConfigLoader.clear` should be called
         to erase any previous state.
         """
         self.clear()
         return self.config
 
 
 class FileConfigLoader(ConfigLoader):
     """A base class for file based configurations.
 
     As we add more file based config loaders, the common logic should go
     here.
     """
 
     def __init__(self, filename, path=None, **kw):
         """Build a config loader for a filename and path.
 
         Parameters
         ----------
         filename : str
             The file name of the config file.
         path : str, list, tuple
             The path to search for the config file on, or a sequence of
             paths to try in order.
         """
         super(FileConfigLoader, self).__init__(**kw)
         self.filename = filename
         self.path = path
         self.full_filename = ''
 
     def _find_file(self):
         """Try to find the file by searching the paths."""
         self.full_filename = filefind(self.filename, self.path)
 
 class JSONFileConfigLoader(FileConfigLoader):
     """A Json file loader for config"""
 
     def load_config(self):
         """Load the config from a file and return it as a Config object."""
         self.clear()
         try:
             self._find_file()
         except IOError as e:
             raise ConfigFileNotFound(str(e))
         dct = self._read_file_as_dict()
         self.config = self._convert_to_config(dct)
         return self.config
 
     def _read_file_as_dict(self):
         with open(self.full_filename) as f:
             return json.load(f)
 
     def _convert_to_config(self, dictionary):
         if 'version' in dictionary:
             version = dictionary.pop('version')
         else:
             version = 1
             self.log.warn("Unrecognized JSON config file version, assuming version {}".format(version))
 
         if version == 1:
             return Config(dictionary)
         else:
             raise ValueError('Unknown version of JSON config file: {version}'.format(version=version))
 
 
 class PyFileConfigLoader(FileConfigLoader):
     """A config loader for pure python files.
 
     This is responsible for locating a Python config file by filename and
     path, then executing it to construct a Config object.
     """
 
     def load_config(self):
         """Load the config from a file and return it as a Config object."""
         self.clear()
         try:
             self._find_file()
         except IOError as e:
             raise ConfigFileNotFound(str(e))
         self._read_file_as_dict()
         return self.config
 
 
     def _read_file_as_dict(self):
         """Load the config file into self.config, with recursive loading."""
         # This closure is made available in the namespace that is used
         # to exec the config file.  It allows users to call
         # load_subconfig('myconfig.py') to load config files recursively.
         # It needs to be a closure because it has references to self.path
         # and self.config.  The sub-config is loaded with the same path
         # as the parent, but it uses an empty config which is then merged
         # with the parents.
 
         # If a profile is specified, the config file will be loaded
         # from that profile
 
         def load_subconfig(fname, profile=None):
             # import here to prevent circular imports
             from IPython.core.profiledir import ProfileDir, ProfileDirError
             if profile is not None:
                 try:
                     profile_dir = ProfileDir.find_profile_dir_by_name(
                             get_ipython_dir(),
                             profile,
                     )
                 except ProfileDirError:
                     return
                 path = profile_dir.location
             else:
                 path = self.path
             loader = PyFileConfigLoader(fname, path)
             try:
                 sub_config = loader.load_config()
             except ConfigFileNotFound:
                 # Pass silently if the sub config is not there. This happens
                 # when a user s using a profile, but not the default config.
                 pass
             else:
                 self.config.merge(sub_config)
 
         # Again, this needs to be a closure and should be used in config
         # files to get the config being loaded.
         def get_config():
             return self.config
 
         namespace = dict(
             load_subconfig=load_subconfig,
             get_config=get_config,
             __file__=self.full_filename,
         )
         fs_encoding = sys.getfilesystemencoding() or 'ascii'
         conf_filename = self.full_filename.encode(fs_encoding)
         py3compat.execfile(conf_filename, namespace)
 
 
 class CommandLineConfigLoader(ConfigLoader):
     """A config loader for command line arguments.
 
     As we add more command line based loaders, the common logic should go
     here.
     """
 
     def _exec_config_str(self, lhs, rhs):
         """execute self.config.<lhs> = <rhs>
         
         * expands ~ with expanduser
         * tries to assign with raw eval, otherwise assigns with just the string,
           allowing `--C.a=foobar` and `--C.a="foobar"` to be equivalent.  *Not*
           equivalent are `--C.a=4` and `--C.a='4'`.
         """
         rhs = os.path.expanduser(rhs)
         try:
             # Try to see if regular Python syntax will work. This
             # won't handle strings as the quote marks are removed
             # by the system shell.
             value = eval(rhs)
         except (NameError, SyntaxError):
             # This case happens if the rhs is a string.
             value = rhs
 
         exec(u'self.config.%s = value' % lhs)
 
     def _load_flag(self, cfg):
         """update self.config from a flag, which can be a dict or Config"""
         if isinstance(cfg, (dict, Config)):
             # don't clobber whole config sections, update
             # each section from config:
             for sec,c in iteritems(cfg):
                 self.config[sec].update(c)
         else:
             raise TypeError("Invalid flag: %r" % cfg)
 
 # raw --identifier=value pattern
 # but *also* accept '-' as wordsep, for aliases
 # accepts:  --foo=a
 #           --Class.trait=value
 #           --alias-name=value
 # rejects:  -foo=value
 #           --foo
 #           --Class.trait
 kv_pattern = re.compile(r'\-\-[A-Za-z][\w\-]*(\.[\w\-]+)*\=.*')
 
 # just flags, no assignments, with two *or one* leading '-'
 # accepts:  --foo
 #           -foo-bar-again
 # rejects:  --anything=anything
 #           --two.word
 
 flag_pattern = re.compile(r'\-\-?\w+[\-\w]*$')
 
 class KeyValueConfigLoader(CommandLineConfigLoader):
     """A config loader that loads key value pairs from the command line.
 
     This allows command line options to be gives in the following form::
 
         ipython --profile="foo" --InteractiveShell.autocall=False
     """
 
     def __init__(self, argv=None, aliases=None, flags=None, **kw):
         """Create a key value pair config loader.
 
         Parameters
         ----------
         argv : list
             A list that has the form of sys.argv[1:] which has unicode
             elements of the form u"key=value". If this is None (default),
             then sys.argv[1:] will be used.
         aliases : dict
             A dict of aliases for configurable traits.
             Keys are the short aliases, Values are the resolved trait.
             Of the form: `{'alias' : 'Configurable.trait'}`
         flags : dict
             A dict of flags, keyed by str name. Vaues can be Config objects,
             dicts, or "key=value" strings.  If Config or dict, when the flag
             is triggered, The flag is loaded as `self.config.update(m)`.
 
         Returns
         -------
         config : Config
             The resulting Config object.
 
         Examples
         --------
 
             >>> from IPython.config.loader import KeyValueConfigLoader
             >>> cl = KeyValueConfigLoader()
             >>> d = cl.load_config(["--A.name='brian'","--B.number=0"])
             >>> sorted(d.items())
             [('A', {'name': 'brian'}), ('B', {'number': 0})]
         """
         super(KeyValueConfigLoader, self).__init__(**kw)
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
 
 
     def clear(self):
         super(KeyValueConfigLoader, self).clear()
         self.extra_args = []
 
 
     def _decode_argv(self, argv, enc=None):
-        """decode argv if bytes, using stin.encoding, falling back on default enc"""
+        """decode argv if bytes, using stdin.encoding, falling back on default enc"""
         uargv = []
         if enc is None:
             enc = DEFAULT_ENCODING
         for arg in argv:
             if not isinstance(arg, unicode_type):
                 # only decode if not already decoded
                 arg = arg.decode(enc)
             uargv.append(arg)
         return uargv
 
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse the configuration and generate the Config object.
 
         After loading, any arguments that are not key-value or
         flags will be stored in self.extra_args - a list of
         unparsed command-line arguments.  This is used for
         arguments such as input files or subcommands.
 
         Parameters
         ----------
         argv : list, optional
             A list that has the form of sys.argv[1:] which has unicode
             elements of the form u"key=value". If this is None (default),
             then self.argv will be used.
         aliases : dict
             A dict of aliases for configurable traits.
             Keys are the short aliases, Values are the resolved trait.
             Of the form: `{'alias' : 'Configurable.trait'}`
         flags : dict
             A dict of flags, keyed by str name. Values can be Config objects
             or dicts.  When the flag is triggered, The config is loaded as
             `self.config.update(cfg)`.
         """
         self.clear()
         if argv is None:
             argv = self.argv
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
 
         # ensure argv is a list of unicode strings:
         uargv = self._decode_argv(argv)
         for idx,raw in enumerate(uargv):
             # strip leading '-'
             item = raw.lstrip('-')
 
             if raw == '--':
                 # don't parse arguments after '--'
                 # this is useful for relaying arguments to scripts, e.g.
                 # ipython -i foo.py --matplotlib=qt -- args after '--' go-to-foo.py
                 self.extra_args.extend(uargv[idx+1:])
                 break
 
             if kv_pattern.match(raw):
                 lhs,rhs = item.split('=',1)
                 # Substitute longnames for aliases.
                 if lhs in aliases:
                     lhs = aliases[lhs]
                 if '.' not in lhs:
                     # probably a mistyped alias, but not technically illegal
                     self.log.warn("Unrecognized alias: '%s', it will probably have no effect.", raw)
                 try:
                     self._exec_config_str(lhs, rhs)
                 except Exception:
                     raise ArgumentError("Invalid argument: '%s'" % raw)
 
             elif flag_pattern.match(raw):
                 if item in flags:
                     cfg,help = flags[item]
                     self._load_flag(cfg)
                 else:
                     raise ArgumentError("Unrecognized flag: '%s'"%raw)
             elif raw.startswith('-'):
                 kv = '--'+item
                 if kv_pattern.match(kv):
                     raise ArgumentError("Invalid argument: '%s', did you mean '%s'?"%(raw, kv))
                 else:
                     raise ArgumentError("Invalid argument: '%s'"%raw)
             else:
                 # keep all args that aren't valid in a list,
                 # in case our parent knows what to do with them.
                 self.extra_args.append(item)
         return self.config
 
 class ArgParseConfigLoader(CommandLineConfigLoader):
     """A loader that uses the argparse module to load from the command line."""
 
     def __init__(self, argv=None, aliases=None, flags=None, log=None,  *parser_args, **parser_kw):
         """Create a config loader for use with argparse.
 
         Parameters
         ----------
 
         argv : optional, list
           If given, used to read command-line arguments from, otherwise
           sys.argv[1:] is used.
 
         parser_args : tuple
           A tuple of positional arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         parser_kw : dict
           A tuple of keyword arguments that will be passed to the
           constructor of :class:`argparse.ArgumentParser`.
 
         Returns
         -------
         config : Config
             The resulting Config object.
         """
         super(CommandLineConfigLoader, self).__init__(log=log)
         self.clear()
         if argv is None:
             argv = sys.argv[1:]
         self.argv = argv
         self.aliases = aliases or {}
         self.flags = flags or {}
 
         self.parser_args = parser_args
         self.version = parser_kw.pop("version", None)
         kwargs = dict(argument_default=argparse.SUPPRESS)
         kwargs.update(parser_kw)
         self.parser_kw = kwargs
 
     def load_config(self, argv=None, aliases=None, flags=None):
         """Parse command line arguments and return as a Config object.
 
         Parameters
         ----------
 
         args : optional, list
           If given, a list with the structure of sys.argv[1:] to parse
           arguments from. If not given, the instance's self.argv attribute
           (given at construction time) is used."""
         self.clear()
         if argv is None:
             argv = self.argv
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
         self._create_parser(aliases, flags)
         self._parse_args(argv)
         self._convert_to_config()
         return self.config
 
     def get_extra_args(self):
         if hasattr(self, 'extra_args'):
             return self.extra_args
         else:
             return []
 
     def _create_parser(self, aliases=None, flags=None):
         self.parser = ArgumentParser(*self.parser_args, **self.parser_kw)
         self._add_arguments(aliases, flags)
 
     def _add_arguments(self, aliases=None, flags=None):
         raise NotImplementedError("subclasses must implement _add_arguments")
 
     def _parse_args(self, args):
         """self.parser->self.parsed_data"""
         # decode sys.argv to support unicode command-line options
         enc = DEFAULT_ENCODING
         uargs = [py3compat.cast_unicode(a, enc) for a in args]
         self.parsed_data, self.extra_args = self.parser.parse_known_args(uargs)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config"""
         for k, v in iteritems(vars(self.parsed_data)):
             exec("self.config.%s = v"%k, locals(), globals())
 
 class KVArgParseConfigLoader(ArgParseConfigLoader):
     """A config loader that loads aliases and flags with argparse,
     but will use KVLoader for the rest.  This allows better parsing
     of common args, such as `ipython -c 'print 5'`, but still gets
     arbitrary config with `ipython --InteractiveShell.use_readline=False`"""
 
     def _add_arguments(self, aliases=None, flags=None):
         self.alias_flags = {}
         # print aliases, flags
         if aliases is None:
             aliases = self.aliases
         if flags is None:
             flags = self.flags
         paa = self.parser.add_argument
         for key,value in iteritems(aliases):
             if key in flags:
                 # flags
                 nargs = '?'
             else:
                 nargs = None
             if len(key) is 1:
                 paa('-'+key, '--'+key, type=unicode_type, dest=value, nargs=nargs)
             else:
                 paa('--'+key, type=unicode_type, dest=value, nargs=nargs)
         for key, (value, help) in iteritems(flags):
             if key in self.aliases:
                 #
                 self.alias_flags[self.aliases[key]] = value
                 continue
             if len(key) is 1:
                 paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value)
             else:
                 paa('--'+key, action='append_const', dest='_flags', const=value)
 
     def _convert_to_config(self):
         """self.parsed_data->self.config, parse unrecognized extra args via KVLoader."""
         # remove subconfigs list from namespace before transforming the Namespace
         if '_flags' in self.parsed_data:
             subcs = self.parsed_data._flags
             del self.parsed_data._flags
         else:
             subcs = []
 
         for k, v in iteritems(vars(self.parsed_data)):
             if v is None:
                 # it was a flag that shares the name of an alias
                 subcs.append(self.alias_flags[k])
             else:
                 # eval the KV assignment
                 self._exec_config_str(k, v)
 
         for subc in subcs:
             self._load_flag(subc)
 
         if self.extra_args:
             sub_parser = KeyValueConfigLoader(log=self.log)
             sub_parser.load_config(self.extra_args)
             self.config.merge(sub_parser.config)
             self.extra_args = sub_parser.extra_args
 
 
 def load_pyconfig_files(config_files, path):
     """Load multiple Python config files, merging each of them in turn.
 
     Parameters
     ==========
     config_files : list of str
         List of config files names to load and merge into the config.
     path : unicode
         The full path to the location of the config files.
     """
     config = Config()
     for cf in config_files:
         loader = PyFileConfigLoader(cf, path=path)
         try:
             next_config = loader.load_config()
         except ConfigFileNotFound:
             pass
         except:
             raise
         else:
             config.merge(next_config)
     return config

IPython/config/tests/test_loader.py

@@ -1,396 +1,404 b''
 # encoding: utf-8
-"""
-Tests for IPython.config.loader
-
-Authors:
-
-* Brian Granger
-* Fernando Perez (design help)
-"""
-
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008 The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
+"""Tests for IPython.config.loader"""
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import os
 import pickle
 import sys
-import json
 
 from tempfile import mkstemp
 from unittest import TestCase
 
 from nose import SkipTest
 import nose.tools as nt
 
 
 
 from IPython.config.loader import (
     Config,
     LazyConfigValue,
     PyFileConfigLoader,
     JSONFileConfigLoader,
     KeyValueConfigLoader,
     ArgParseConfigLoader,
     KVArgParseConfigLoader,
     ConfigError,
 )
 
-#-----------------------------------------------------------------------------
-# Actual tests
-#-----------------------------------------------------------------------------
-
 
 pyfile = """
 c = get_config()
 c.a=10
 c.b=20
 c.Foo.Bar.value=10
 c.Foo.Bam.value=list(range(10))  # list() is just so it's the same on Python 3
 c.D.C.value='hi there'
 """
 
 json1file = """
 {
   "version": 1,
   "a": 10,
   "b": 20,
   "Foo": {
     "Bam": {
       "value": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ]
     },
     "Bar": {
       "value": 10
     }
   },
   "D": {
     "C": {
       "value": "hi there"
     }
   }
 }
 """
 
 # should not load
 json2file = """
 {
   "version": 2
 }
 """
 
 import logging
 log = logging.getLogger('devnull')
 log.setLevel(0)
 
 class TestFileCL(TestCase):
 
     def _check_conf(self, config):
         self.assertEqual(config.a, 10)
         self.assertEqual(config.b, 20)
         self.assertEqual(config.Foo.Bar.value, 10)
         self.assertEqual(config.Foo.Bam.value, list(range(10)))
         self.assertEqual(config.D.C.value, 'hi there')
 
     def test_python(self):
         fd, fname = mkstemp('.py')
         f = os.fdopen(fd, 'w')
         f.write(pyfile)
         f.close()
         # Unlink the file
         cl = PyFileConfigLoader(fname, log=log)
         config = cl.load_config()
         self._check_conf(config)
 
     def test_json(self):
         fd, fname = mkstemp('.json')
         f = os.fdopen(fd, 'w')
         f.write(json1file)
         f.close()
         # Unlink the file
         cl = JSONFileConfigLoader(fname, log=log)
         config = cl.load_config()
         self._check_conf(config)
 
+    def test_collision(self):
+        a = Config()
+        b = Config()
+        self.assertEqual(a.collisions(b), {})
+        a.A.trait1 = 1
+        b.A.trait2 = 2
+        self.assertEqual(a.collisions(b), {})
+        b.A.trait1 = 1
+        self.assertEqual(a.collisions(b), {})
+        b.A.trait1 = 0
+        self.assertEqual(a.collisions(b), {
+            'A': {
+                'trait1': "1 ignored, using 0",
+            }
+        })
+        self.assertEqual(b.collisions(a), {
+            'A': {
+                'trait1': "0 ignored, using 1",
+            }
+        })
+        a.A.trait2 = 3
+        self.assertEqual(b.collisions(a), {
+            'A': {
+                'trait1': "0 ignored, using 1",
+                'trait2': "2 ignored, using 3",
+            }
+        })
+
     def test_v2raise(self):
         fd, fname = mkstemp('.json')
         f = os.fdopen(fd, 'w')
         f.write(json2file)
         f.close()
         # Unlink the file
         cl = JSONFileConfigLoader(fname, log=log)
         with nt.assert_raises(ValueError):
             cl.load_config()
 
 
 class MyLoader1(ArgParseConfigLoader):
     def _add_arguments(self, aliases=None, flags=None):
         p = self.parser
         p.add_argument('-f', '--foo', dest='Global.foo', type=str)
         p.add_argument('-b', dest='MyClass.bar', type=int)
         p.add_argument('-n', dest='n', action='store_true')
         p.add_argument('Global.bam', type=str)
 
 class MyLoader2(ArgParseConfigLoader):
     def _add_arguments(self, aliases=None, flags=None):
         subparsers = self.parser.add_subparsers(dest='subparser_name')
         subparser1 = subparsers.add_parser('1')
         subparser1.add_argument('-x',dest='Global.x')
         subparser2 = subparsers.add_parser('2')
         subparser2.add_argument('y')
 
 class TestArgParseCL(TestCase):
 
     def test_basic(self):
         cl = MyLoader1()
         config = cl.load_config('-f hi -b 10 -n wow'.split())
         self.assertEqual(config.Global.foo, 'hi')
         self.assertEqual(config.MyClass.bar, 10)
         self.assertEqual(config.n, True)
         self.assertEqual(config.Global.bam, 'wow')
         config = cl.load_config(['wow'])
         self.assertEqual(list(config.keys()), ['Global'])
         self.assertEqual(list(config.Global.keys()), ['bam'])
         self.assertEqual(config.Global.bam, 'wow')
 
     def test_add_arguments(self):
         cl = MyLoader2()
         config = cl.load_config('2 frobble'.split())
         self.assertEqual(config.subparser_name, '2')
         self.assertEqual(config.y, 'frobble')
         config = cl.load_config('1 -x frobble'.split())
         self.assertEqual(config.subparser_name, '1')
         self.assertEqual(config.Global.x, 'frobble')
 
     def test_argv(self):
         cl = MyLoader1(argv='-f hi -b 10 -n wow'.split())
         config = cl.load_config()
         self.assertEqual(config.Global.foo, 'hi')
         self.assertEqual(config.MyClass.bar, 10)
         self.assertEqual(config.n, True)
         self.assertEqual(config.Global.bam, 'wow')
 
 
 class TestKeyValueCL(TestCase):
     klass = KeyValueConfigLoader
 
     def test_basic(self):
         cl = self.klass(log=log)
         argv = ['--'+s.strip('c.') for s in pyfile.split('\n')[2:-1]]
         config = cl.load_config(argv)
         self.assertEqual(config.a, 10)
         self.assertEqual(config.b, 20)
         self.assertEqual(config.Foo.Bar.value, 10)
         self.assertEqual(config.Foo.Bam.value, list(range(10)))
         self.assertEqual(config.D.C.value, 'hi there')
     
     def test_expanduser(self):
         cl = self.klass(log=log)
         argv = ['--a=~/1/2/3', '--b=~', '--c=~/', '--d="~/"']
         config = cl.load_config(argv)
         self.assertEqual(config.a, os.path.expanduser('~/1/2/3'))
         self.assertEqual(config.b, os.path.expanduser('~'))
         self.assertEqual(config.c, os.path.expanduser('~/'))
         self.assertEqual(config.d, '~/')
     
     def test_extra_args(self):
         cl = self.klass(log=log)
         config = cl.load_config(['--a=5', 'b', '--c=10', 'd'])
         self.assertEqual(cl.extra_args, ['b', 'd'])
         self.assertEqual(config.a, 5)
         self.assertEqual(config.c, 10)
         config = cl.load_config(['--', '--a=5', '--c=10'])
         self.assertEqual(cl.extra_args, ['--a=5', '--c=10'])
     
     def test_unicode_args(self):
         cl = self.klass(log=log)
         argv = [u'--a=épsîlön']
         config = cl.load_config(argv)
         self.assertEqual(config.a, u'épsîlön')
     
     def test_unicode_bytes_args(self):
         uarg = u'--a=é'
         try:
             barg = uarg.encode(sys.stdin.encoding)
         except (TypeError, UnicodeEncodeError):
             raise SkipTest("sys.stdin.encoding can't handle 'é'")
         
         cl = self.klass(log=log)
         config = cl.load_config([barg])
         self.assertEqual(config.a, u'é')
     
     def test_unicode_alias(self):
         cl = self.klass(log=log)
         argv = [u'--a=épsîlön']
         config = cl.load_config(argv, aliases=dict(a='A.a'))
         self.assertEqual(config.A.a, u'épsîlön')
 
 
 class TestArgParseKVCL(TestKeyValueCL):
     klass = KVArgParseConfigLoader
 
     def test_expanduser2(self):
         cl = self.klass(log=log)
         argv = ['-a', '~/1/2/3', '--b', "'~/1/2/3'"]
         config = cl.load_config(argv, aliases=dict(a='A.a', b='A.b'))
         self.assertEqual(config.A.a, os.path.expanduser('~/1/2/3'))
         self.assertEqual(config.A.b, '~/1/2/3')
     
     def test_eval(self):
         cl = self.klass(log=log)
         argv = ['-c', 'a=5']
         config = cl.load_config(argv, aliases=dict(c='A.c'))
         self.assertEqual(config.A.c, u"a=5")
     
 
 class TestConfig(TestCase):
 
     def test_setget(self):
         c = Config()
         c.a = 10
         self.assertEqual(c.a, 10)
         self.assertEqual('b' in c, False)
 
     def test_auto_section(self):
         c = Config()
         self.assertNotIn('A', c)
         assert not c._has_section('A')
         A = c.A
         A.foo = 'hi there'
         self.assertIn('A', c)
         assert c._has_section('A')
         self.assertEqual(c.A.foo, 'hi there')
         del c.A
         self.assertEqual(c.A, Config())
 
     def test_merge_doesnt_exist(self):
         c1 = Config()
         c2 = Config()
         c2.bar = 10
         c2.Foo.bar = 10
         c1.merge(c2)
         self.assertEqual(c1.Foo.bar, 10)
         self.assertEqual(c1.bar, 10)
         c2.Bar.bar = 10
         c1.merge(c2)
         self.assertEqual(c1.Bar.bar, 10)
 
     def test_merge_exists(self):
         c1 = Config()
         c2 = Config()
         c1.Foo.bar = 10
         c1.Foo.bam = 30
         c2.Foo.bar = 20
         c2.Foo.wow = 40
         c1.merge(c2)
         self.assertEqual(c1.Foo.bam, 30)
         self.assertEqual(c1.Foo.bar, 20)
         self.assertEqual(c1.Foo.wow, 40)
         c2.Foo.Bam.bam = 10
         c1.merge(c2)
         self.assertEqual(c1.Foo.Bam.bam, 10)
 
     def test_deepcopy(self):
         c1 = Config()
         c1.Foo.bar = 10
         c1.Foo.bam = 30
         c1.a = 'asdf'
         c1.b = range(10)
         import copy
         c2 = copy.deepcopy(c1)
         self.assertEqual(c1, c2)
         self.assertTrue(c1 is not c2)
         self.assertTrue(c1.Foo is not c2.Foo)
 
     def test_builtin(self):
         c1 = Config()
         c1.format = "json"
     
     def test_fromdict(self):
         c1 = Config({'Foo' : {'bar' : 1}})
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
     
     def test_fromdictmerge(self):
         c1 = Config()
         c2 = Config({'Foo' : {'bar' : 1}})
         c1.merge(c2)
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
 
     def test_fromdictmerge2(self):
         c1 = Config({'Foo' : {'baz' : 2}})
         c2 = Config({'Foo' : {'bar' : 1}})
         c1.merge(c2)
         self.assertEqual(c1.Foo.__class__, Config)
         self.assertEqual(c1.Foo.bar, 1)
         self.assertEqual(c1.Foo.baz, 2)
         self.assertNotIn('baz', c2.Foo)
     
     def test_contains(self):
         c1 = Config({'Foo' : {'baz' : 2}})
         c2 = Config({'Foo' : {'bar' : 1}})
         self.assertIn('Foo', c1)
         self.assertIn('Foo.baz', c1)
         self.assertIn('Foo.bar', c2)
         self.assertNotIn('Foo.bar', c1)
     
     def test_pickle_config(self):
         cfg = Config()
         cfg.Foo.bar = 1
         pcfg = pickle.dumps(cfg)
         cfg2 = pickle.loads(pcfg)
         self.assertEqual(cfg2, cfg)
     
     def test_getattr_section(self):
         cfg = Config()
         self.assertNotIn('Foo', cfg)
         Foo = cfg.Foo
         assert isinstance(Foo, Config)
         self.assertIn('Foo', cfg)
 
     def test_getitem_section(self):
         cfg = Config()
         self.assertNotIn('Foo', cfg)
         Foo = cfg['Foo']
         assert isinstance(Foo, Config)
         self.assertIn('Foo', cfg)
 
     def test_getattr_not_section(self):
         cfg = Config()
         self.assertNotIn('foo', cfg)
         foo = cfg.foo
         assert isinstance(foo, LazyConfigValue)
         self.assertIn('foo', cfg)
 
     def test_getattr_private_missing(self):
         cfg = Config()
         self.assertNotIn('_repr_html_', cfg)
         with self.assertRaises(AttributeError):
             _ = cfg._repr_html_
         self.assertNotIn('_repr_html_', cfg)
         self.assertEqual(len(cfg), 0)
 
     def test_getitem_not_section(self):
         cfg = Config()
         self.assertNotIn('foo', cfg)
         foo = cfg['foo']
         assert isinstance(foo, LazyConfigValue)
         self.assertIn('foo', cfg)
     
     def test_merge_copies(self):
         c = Config()
         c2 = Config()
         c2.Foo.trait = []
         c.merge(c2)
         c2.Foo.trait.append(1)
         self.assertIsNot(c.Foo, c2.Foo)
         self.assertEqual(c.Foo.trait, [])
         self.assertEqual(c2.Foo.trait, [1])
 

IPython/html/base/handlers.py

@@ -1,507 +1,514 b''
 """Base Tornado handlers for the notebook server."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import functools
 import json
 import logging
 import os
 import re
 import sys
 import traceback
 try:
     # py3
     from http.client import responses
 except ImportError:
     from httplib import responses
 
 from jinja2 import TemplateNotFound
 from tornado import web
 
 try:
     from tornado.log import app_log
 except ImportError:
     app_log = logging.getLogger()
 
 import IPython
 from IPython.utils.sysinfo import get_sys_info
 
 from IPython.config import Application
 from IPython.utils.path import filefind
 from IPython.utils.py3compat import string_types
 from IPython.html.utils import is_hidden, url_path_join, url_escape
 
+from IPython.html.services.security import csp_report_uri
+
 #-----------------------------------------------------------------------------
 # Top-level handlers
 #-----------------------------------------------------------------------------
 non_alphanum = re.compile(r'[^A-Za-z0-9]')
 
 sys_info = json.dumps(get_sys_info())
 
 class AuthenticatedHandler(web.RequestHandler):
     """A RequestHandler with an authenticated user."""
 
     def set_default_headers(self):
         headers = self.settings.get('headers', {})
 
-        if "X-Frame-Options" not in headers:
-            headers["X-Frame-Options"] = "SAMEORIGIN"
+        if "Content-Security-Policy" not in headers:
+            headers["Content-Security-Policy"] = (
+                    "frame-ancestors 'self'; "
+                    # Make sure the report-uri is relative to the base_url
+                    "report-uri " + url_path_join(self.base_url, csp_report_uri) + ";"
+            )
 
+        # Allow for overriding headers
         for header_name,value in headers.items() :
             try:
                 self.set_header(header_name, value)
-            except Exception:
+            except Exception as e:
                 # tornado raise Exception (not a subclass)
                 # if method is unsupported (websocket and Access-Control-Allow-Origin
                 # for example, so just ignore)
-                pass
+                self.log.debug(e)
     
     def clear_login_cookie(self):
         self.clear_cookie(self.cookie_name)
     
     def get_current_user(self):
         user_id = self.get_secure_cookie(self.cookie_name)
         # For now the user_id should not return empty, but it could eventually
         if user_id == '':
             user_id = 'anonymous'
         if user_id is None:
             # prevent extra Invalid cookie sig warnings:
             self.clear_login_cookie()
             if not self.login_available:
                 user_id = 'anonymous'
         return user_id
 
     @property
     def cookie_name(self):
         default_cookie_name = non_alphanum.sub('-', 'username-{}'.format(
             self.request.host
         ))
         return self.settings.get('cookie_name', default_cookie_name)
     
     @property
     def password(self):
         """our password"""
         return self.settings.get('password', '')
     
     @property
     def logged_in(self):
         """Is a user currently logged in?
 
         """
         user = self.get_current_user()
         return (user and not user == 'anonymous')
 
     @property
     def login_available(self):
         """May a user proceed to log in?
 
         This returns True if login capability is available, irrespective of
         whether the user is already logged in or not.
 
         """
         return bool(self.settings.get('password', ''))
 
 
 class IPythonHandler(AuthenticatedHandler):
     """IPython-specific extensions to authenticated handling
     
     Mostly property shortcuts to IPython-specific settings.
     """
     
     @property
     def config(self):
         return self.settings.get('config', None)
     
     @property
     def log(self):
         """use the IPython log by default, falling back on tornado's logger"""
         if Application.initialized():
             return Application.instance().log
         else:
             return app_log
     
     #---------------------------------------------------------------
     # URLs
     #---------------------------------------------------------------
     
     @property
     def version_hash(self):
         """The version hash to use for cache hints for static files"""
         return self.settings.get('version_hash', '')
     
     @property
     def mathjax_url(self):
         return self.settings.get('mathjax_url', '')
     
     @property
     def base_url(self):
         return self.settings.get('base_url', '/')
 
     @property
     def ws_url(self):
         return self.settings.get('websocket_url', '')
 
     @property
     def contents_js_source(self):
         self.log.debug("Using contents: %s", self.settings.get('contents_js_source',
             'services/contents'))
         return self.settings.get('contents_js_source', 'services/contents')
     
     #---------------------------------------------------------------
     # Manager objects
     #---------------------------------------------------------------
     
     @property
     def kernel_manager(self):
         return self.settings['kernel_manager']
 
     @property
     def contents_manager(self):
         return self.settings['contents_manager']
     
     @property
     def cluster_manager(self):
         return self.settings['cluster_manager']
     
     @property
     def session_manager(self):
         return self.settings['session_manager']
     
     @property
     def terminal_manager(self):
         return self.settings['terminal_manager']
     
     @property
     def kernel_spec_manager(self):
         return self.settings['kernel_spec_manager']
 
     @property
     def config_manager(self):
         return self.settings['config_manager']
 
     #---------------------------------------------------------------
     # CORS
     #---------------------------------------------------------------
     
     @property
     def allow_origin(self):
         """Normal Access-Control-Allow-Origin"""
         return self.settings.get('allow_origin', '')
     
     @property
     def allow_origin_pat(self):
         """Regular expression version of allow_origin"""
         return self.settings.get('allow_origin_pat', None)
     
     @property
     def allow_credentials(self):
         """Whether to set Access-Control-Allow-Credentials"""
         return self.settings.get('allow_credentials', False)
     
     def set_default_headers(self):
         """Add CORS headers, if defined"""
         super(IPythonHandler, self).set_default_headers()
         if self.allow_origin:
             self.set_header("Access-Control-Allow-Origin", self.allow_origin)
         elif self.allow_origin_pat:
             origin = self.get_origin()
             if origin and self.allow_origin_pat.match(origin):
                 self.set_header("Access-Control-Allow-Origin", origin)
         if self.allow_credentials:
             self.set_header("Access-Control-Allow-Credentials", 'true')
     
     def get_origin(self):
         # Handle WebSocket Origin naming convention differences
         # The difference between version 8 and 13 is that in 8 the
         # client sends a "Sec-Websocket-Origin" header and in 13 it's
         # simply "Origin".
         if "Origin" in self.request.headers:
             origin = self.request.headers.get("Origin")
         else:
             origin = self.request.headers.get("Sec-Websocket-Origin", None)
         return origin
     
     #---------------------------------------------------------------
     # template rendering
     #---------------------------------------------------------------
     
     def get_template(self, name):
         """Return the jinja template object for a given name"""
         return self.settings['jinja2_env'].get_template(name)
     
     def render_template(self, name, **ns):
         ns.update(self.template_namespace)
         template = self.get_template(name)
         return template.render(**ns)
     
     @property
     def template_namespace(self):
         return dict(
             base_url=self.base_url,
             ws_url=self.ws_url,
             logged_in=self.logged_in,
             login_available=self.login_available,
             static_url=self.static_url,
             sys_info=sys_info,
             contents_js_source=self.contents_js_source,
             version_hash=self.version_hash,
         )
     
     def get_json_body(self):
         """Return the body of the request as JSON data."""
         if not self.request.body:
             return None
         # Do we need to call body.decode('utf-8') here?
         body = self.request.body.strip().decode(u'utf-8')
         try:
             model = json.loads(body)
         except Exception:
             self.log.debug("Bad JSON: %r", body)
             self.log.error("Couldn't parse JSON", exc_info=True)
             raise web.HTTPError(400, u'Invalid JSON in body of request')
         return model
 
     def write_error(self, status_code, **kwargs):
         """render custom error pages"""
         exc_info = kwargs.get('exc_info')
         message = ''
         status_message = responses.get(status_code, 'Unknown HTTP Error')
         if exc_info:
             exception = exc_info[1]
             # get the custom message, if defined
             try:
                 message = exception.log_message % exception.args
             except Exception:
                 pass
             
             # construct the custom reason, if defined
             reason = getattr(exception, 'reason', '')
             if reason:
                 status_message = reason
         
         # build template namespace
         ns = dict(
             status_code=status_code,
             status_message=status_message,
             message=message,
             exception=exception,
         )
         
         self.set_header('Content-Type', 'text/html')
         # render the template
         try:
             html = self.render_template('%s.html' % status_code, **ns)
         except TemplateNotFound:
             self.log.debug("No template for %d", status_code)
             html = self.render_template('error.html', **ns)
         
         self.write(html)
         
 
 
 class Template404(IPythonHandler):
     """Render our 404 template"""
     def prepare(self):
         raise web.HTTPError(404)
 
 
 class AuthenticatedFileHandler(IPythonHandler, web.StaticFileHandler):
     """static files should only be accessible when logged in"""
 
     @web.authenticated
     def get(self, path):
         if os.path.splitext(path)[1] == '.ipynb':
             name = path.rsplit('/', 1)[-1]
             self.set_header('Content-Type', 'application/json')
             self.set_header('Content-Disposition','attachment; filename="%s"' % name)
         
         return web.StaticFileHandler.get(self, path)
     
     def set_headers(self):
         super(AuthenticatedFileHandler, self).set_headers()
         # disable browser caching, rely on 304 replies for savings
         if "v" not in self.request.arguments:
             self.add_header("Cache-Control", "no-cache")
     
     def compute_etag(self):
         return None
     
     def validate_absolute_path(self, root, absolute_path):
         """Validate and return the absolute path.
         
         Requires tornado 3.1
         
         Adding to tornado's own handling, forbids the serving of hidden files.
         """
         abs_path = super(AuthenticatedFileHandler, self).validate_absolute_path(root, absolute_path)
         abs_root = os.path.abspath(root)
         if is_hidden(abs_path, abs_root):
             self.log.info("Refusing to serve hidden file, via 404 Error")
             raise web.HTTPError(404)
         return abs_path
 
 
 def json_errors(method):
     """Decorate methods with this to return GitHub style JSON errors.
     
     This should be used on any JSON API on any handler method that can raise HTTPErrors.
     
     This will grab the latest HTTPError exception using sys.exc_info
     and then:
     
     1. Set the HTTP status code based on the HTTPError
     2. Create and return a JSON body with a message field describing
        the error in a human readable form.
     """
     @functools.wraps(method)
     def wrapper(self, *args, **kwargs):
         try:
             result = method(self, *args, **kwargs)
         except web.HTTPError as e:
             status = e.status_code
             message = e.log_message
             self.log.warn(message)
             self.set_status(e.status_code)
             self.finish(json.dumps(dict(message=message)))
         except Exception:
             self.log.error("Unhandled error in API request", exc_info=True)
             status = 500
             message = "Unknown server error"
             t, value, tb = sys.exc_info()
             self.set_status(status)
             tb_text = ''.join(traceback.format_exception(t, value, tb))
             reply = dict(message=message, traceback=tb_text)
             self.finish(json.dumps(reply))
         else:
             return result
     return wrapper
 
 
 
 #-----------------------------------------------------------------------------
 # File handler
 #-----------------------------------------------------------------------------
 
 # to minimize subclass changes:
 HTTPError = web.HTTPError
 
 class FileFindHandler(web.StaticFileHandler):
     """subclass of StaticFileHandler for serving files from a search path"""
     
     # cache search results, don't search for files more than once
     _static_paths = {}
     
     def set_headers(self):
         super(FileFindHandler, self).set_headers()
         # disable browser caching, rely on 304 replies for savings
         if "v" not in self.request.arguments or \
                 any(self.request.path.startswith(path) for path in self.no_cache_paths):
             self.add_header("Cache-Control", "no-cache")
     
     def initialize(self, path, default_filename=None, no_cache_paths=None):
         self.no_cache_paths = no_cache_paths or []
         
         if isinstance(path, string_types):
             path = [path]
         
         self.root = tuple(
             os.path.abspath(os.path.expanduser(p)) + os.sep for p in path
         )
         self.default_filename = default_filename
     
     def compute_etag(self):
         return None
     
     @classmethod
     def get_absolute_path(cls, roots, path):
         """locate a file to serve on our static file search path"""
         with cls._lock:
             if path in cls._static_paths:
                 return cls._static_paths[path]
             try:
                 abspath = os.path.abspath(filefind(path, roots))
             except IOError:
                 # IOError means not found
                 return ''
             
             cls._static_paths[path] = abspath
             return abspath
     
     def validate_absolute_path(self, root, absolute_path):
         """check if the file should be served (raises 404, 403, etc.)"""
         if absolute_path == '':
             raise web.HTTPError(404)
         
         for root in self.root:
             if (absolute_path + os.sep).startswith(root):
                 break
         
         return super(FileFindHandler, self).validate_absolute_path(root, absolute_path)
 
 
 class ApiVersionHandler(IPythonHandler):
 
     @json_errors
     def get(self):
         # not authenticated, so give as few info as possible
         self.finish(json.dumps({"version":IPython.__version__}))
 
 
 class TrailingSlashHandler(web.RequestHandler):
     """Simple redirect handler that strips trailing slashes
     
     This should be the first, highest priority handler.
     """
     
     def get(self):
         self.redirect(self.request.uri.rstrip('/'))
     
     post = put = get
 
 
 class FilesRedirectHandler(IPythonHandler):
     """Handler for redirecting relative URLs to the /files/ handler"""
     def get(self, path=''):
         cm = self.contents_manager
         if cm.dir_exists(path):
             # it's a *directory*, redirect to /tree
             url = url_path_join(self.base_url, 'tree', path)
         else:
             orig_path = path
             # otherwise, redirect to /files
             parts = path.split('/')
 
             if not cm.file_exists(path=path) and 'files' in parts:
                 # redirect without files/ iff it would 404
                 # this preserves pre-2.0-style 'files/' links
                 self.log.warn("Deprecated files/ URL: %s", orig_path)
                 parts.remove('files')
                 path = '/'.join(parts)
 
             if not cm.file_exists(path=path):
                 raise web.HTTPError(404)
 
             url = url_path_join(self.base_url, 'files', path)
         url = url_escape(url)
         self.log.debug("Redirecting %s to %s", self.request.path, url)
         self.redirect(url)
 
 
 #-----------------------------------------------------------------------------
 # URL pattern fragments for re-use
 #-----------------------------------------------------------------------------
 
 # path matches any number of `/foo[/bar...]` or just `/` or ''
 path_regex = r"(?P<path>(?:(?:/[^/]+)+|/?))"
 notebook_path_regex = r"(?P<path>(?:/[^/]+)+\.ipynb)"
 
 #-----------------------------------------------------------------------------
 # URL to handler mappings
 #-----------------------------------------------------------------------------
 
 
 default_handlers = [
     (r".*/", TrailingSlashHandler),
     (r"api", ApiVersionHandler)
 ]

IPython/html/notebookapp.py

@@ -1,1041 +1,1041 b''
 # coding: utf-8
 """A tornado based IPython notebook server."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import base64
 import datetime
 import errno
 import io
 import json
 import logging
 import os
 import random
 import re
 import select
 import signal
 import socket
 import sys
 import threading
 import time
 import webbrowser
 
 
 # check for pyzmq 2.1.11
 from IPython.utils.zmqrelated import check_for_zmq
 check_for_zmq('2.1.11', 'IPython.html')
 
 from jinja2 import Environment, FileSystemLoader
 
 # Install the pyzmq ioloop. This has to be done before anything else from
 # tornado is imported.
 from zmq.eventloop import ioloop
 ioloop.install()
 
 # check for tornado 3.1.0
 msg = "The IPython Notebook requires tornado >= 4.0"
 try:
     import tornado
 except ImportError:
     raise ImportError(msg)
 try:
     version_info = tornado.version_info
 except AttributeError:
     raise ImportError(msg + ", but you have < 1.1.0")
 if version_info < (4,0):
     raise ImportError(msg + ", but you have %s" % tornado.version)
 
 from tornado import httpserver
 from tornado import web
 from tornado.log import LogFormatter, app_log, access_log, gen_log
 
 from IPython.html import (
     DEFAULT_STATIC_FILES_PATH,
     DEFAULT_TEMPLATE_PATH_LIST,
 )
 from .base.handlers import Template404
 from .log import log_request
 from .services.kernels.kernelmanager import MappingKernelManager
 from .services.contents.manager import ContentsManager
 from .services.contents.filemanager import FileContentsManager
 from .services.clusters.clustermanager import ClusterManager
 from .services.sessions.sessionmanager import SessionManager
 
 from .base.handlers import AuthenticatedFileHandler, FileFindHandler
 
 from IPython.config import Config
 from IPython.config.application import catch_config_error, boolean_flag
 from IPython.core.application import (
     BaseIPythonApplication, base_flags, base_aliases,
 )
 from IPython.core.profiledir import ProfileDir
 from IPython.kernel import KernelManager
 from IPython.kernel.kernelspec import KernelSpecManager
 from IPython.kernel.zmq.session import default_secure, Session
 from IPython.nbformat.sign import NotebookNotary
 from IPython.utils.importstring import import_item
 from IPython.utils import submodule
 from IPython.utils.process import check_pid
 from IPython.utils.traitlets import (
     Dict, Unicode, Integer, List, Bool, Bytes, Instance,
     DottedObjectName, TraitError,
 )
 from IPython.utils import py3compat
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils.sysinfo import get_sys_info
 
 from .utils import url_path_join
 
 #-----------------------------------------------------------------------------
 # Module globals
 #-----------------------------------------------------------------------------
 
 _examples = """
 ipython notebook                       # start the notebook
 ipython notebook --profile=sympy       # use the sympy profile
 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
 """
 
 #-----------------------------------------------------------------------------
 # Helper functions
 #-----------------------------------------------------------------------------
 
 def random_ports(port, n):
     """Generate a list of n random ports near the given port.
 
     The first 5 ports will be sequential, and the remaining n-5 will be
     randomly selected in the range [port-2*n, port+2*n].
     """
     for i in range(min(5, n)):
         yield port + i
     for i in range(n-5):
         yield max(1, port + random.randint(-2*n, 2*n))
 
 def load_handlers(name):
     """Load the (URL pattern, handler) tuples for each component."""
     name = 'IPython.html.' + name
     mod = __import__(name, fromlist=['default_handlers'])
     return mod.default_handlers
 
 #-----------------------------------------------------------------------------
 # The Tornado web application
 #-----------------------------------------------------------------------------
 
 class NotebookWebApplication(web.Application):
 
     def __init__(self, ipython_app, kernel_manager, contents_manager,
                  cluster_manager, session_manager, kernel_spec_manager,
                  config_manager, log,
                  base_url, default_url, settings_overrides, jinja_env_options):
 
         settings = self.init_settings(
             ipython_app, kernel_manager, contents_manager, cluster_manager,
             session_manager, kernel_spec_manager, config_manager, log, base_url,
             default_url, settings_overrides, jinja_env_options)
         handlers = self.init_handlers(settings)
 
         super(NotebookWebApplication, self).__init__(handlers, **settings)
 
     def init_settings(self, ipython_app, kernel_manager, contents_manager,
                       cluster_manager, session_manager, kernel_spec_manager,
                       config_manager,
                       log, base_url, default_url, settings_overrides,
                       jinja_env_options=None):
 
         _template_path = settings_overrides.get(
             "template_path",
             ipython_app.template_file_path,
         )
         if isinstance(_template_path, str):
             _template_path = (_template_path,)
         template_path = [os.path.expanduser(path) for path in _template_path]
 
         jenv_opt = jinja_env_options if jinja_env_options else {}
         env = Environment(loader=FileSystemLoader(template_path), **jenv_opt)
         
         sys_info = get_sys_info()
         if sys_info['commit_source'] == 'repository':
             # don't cache (rely on 304) when working from master
             version_hash = ''
         else:
             # reset the cache on server restart
             version_hash = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
 
         settings = dict(
             # basics
             log_function=log_request,
             base_url=base_url,
             default_url=default_url,
             template_path=template_path,
             static_path=ipython_app.static_file_path,
             static_handler_class = FileFindHandler,
             static_url_prefix = url_path_join(base_url,'/static/'),
             static_handler_args = {
                 # don't cache custom.js
                 'no_cache_paths': [url_path_join(base_url, 'static', 'custom')],
             },
             version_hash=version_hash,
             
             # authentication
             cookie_secret=ipython_app.cookie_secret,
             login_url=url_path_join(base_url,'/login'),
             password=ipython_app.password,
             
             # managers
             kernel_manager=kernel_manager,
             contents_manager=contents_manager,
             cluster_manager=cluster_manager,
             session_manager=session_manager,
             kernel_spec_manager=kernel_spec_manager,
             config_manager=config_manager,
 
             # IPython stuff
             nbextensions_path = ipython_app.nbextensions_path,
             websocket_url=ipython_app.websocket_url,
             mathjax_url=ipython_app.mathjax_url,
             config=ipython_app.config,
             jinja2_env=env,
             terminals_available=False,  # Set later if terminals are available
         )
 
         # allow custom overrides for the tornado web app.
         settings.update(settings_overrides)
         return settings
 
     def init_handlers(self, settings):
         """Load the (URL pattern, handler) tuples for each component."""
         
         # Order matters. The first handler to match the URL will handle the request.
         handlers = []
         handlers.extend(load_handlers('tree.handlers'))
         handlers.extend(load_handlers('auth.login'))
         handlers.extend(load_handlers('auth.logout'))
         handlers.extend(load_handlers('files.handlers'))
         handlers.extend(load_handlers('notebook.handlers'))
         handlers.extend(load_handlers('nbconvert.handlers'))
         handlers.extend(load_handlers('kernelspecs.handlers'))
         handlers.extend(load_handlers('edit.handlers'))
         handlers.extend(load_handlers('services.config.handlers'))
         handlers.extend(load_handlers('services.kernels.handlers'))
         handlers.extend(load_handlers('services.contents.handlers'))
         handlers.extend(load_handlers('services.clusters.handlers'))
         handlers.extend(load_handlers('services.sessions.handlers'))
         handlers.extend(load_handlers('services.nbconvert.handlers'))
         handlers.extend(load_handlers('services.kernelspecs.handlers'))
-        
+        handlers.extend(load_handlers('services.security.handlers'))
         handlers.append(
             (r"/nbextensions/(.*)", FileFindHandler, {
                 'path': settings['nbextensions_path'],
                 'no_cache_paths': ['/'], # don't cache anything in nbextensions
             }),
         )
         # register base handlers last
         handlers.extend(load_handlers('base.handlers'))
         # set the URL that will be redirected from `/`
         handlers.append(
             (r'/?', web.RedirectHandler, {
                 'url' : url_path_join(settings['base_url'], settings['default_url']),
                 'permanent': False, # want 302, not 301
             })
         )
         # prepend base_url onto the patterns that we match
         new_handlers = []
         for handler in handlers:
             pattern = url_path_join(settings['base_url'], handler[0])
             new_handler = tuple([pattern] + list(handler[1:]))
             new_handlers.append(new_handler)
         # add 404 on the end, which will catch everything that falls through
         new_handlers.append((r'(.*)', Template404))
         return new_handlers
 
 
 class NbserverListApp(BaseIPythonApplication):
     
     description="List currently running notebook servers in this profile."
     
     flags = dict(
         json=({'NbserverListApp': {'json': True}},
               "Produce machine-readable JSON output."),
     )
     
     json = Bool(False, config=True,
           help="If True, each line of output will be a JSON object with the "
                   "details from the server info file.")
 
     def start(self):
         if not self.json:
             print("Currently running servers:")
         for serverinfo in list_running_servers(self.profile):
             if self.json:
                 print(json.dumps(serverinfo))
             else:
                 print(serverinfo['url'], "::", serverinfo['notebook_dir'])
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = dict(base_flags)
 flags['no-browser']=(
     {'NotebookApp' : {'open_browser' : False}},
     "Don't open the notebook in a browser after startup."
 )
 flags['pylab']=(
     {'NotebookApp' : {'pylab' : 'warn'}},
     "DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib."
 )
 flags['no-mathjax']=(
     {'NotebookApp' : {'enable_mathjax' : False}},
     """Disable MathJax
     
     MathJax is the javascript library IPython uses to render math/LaTeX. It is
     very large, so you may want to disable it if you have a slow internet
     connection, or for offline use of the notebook.
     
     When disabled, equations etc. will appear as their untransformed TeX source.
     """
 )
 
 # Add notebook manager flags
 flags.update(boolean_flag('script', 'FileContentsManager.save_script',
                'DEPRECATED, IGNORED',
                'DEPRECATED, IGNORED'))
 
 aliases = dict(base_aliases)
 
 aliases.update({
     'ip': 'NotebookApp.ip',
     'port': 'NotebookApp.port',
     'port-retries': 'NotebookApp.port_retries',
     'transport': 'KernelManager.transport',
     'keyfile': 'NotebookApp.keyfile',
     'certfile': 'NotebookApp.certfile',
     'notebook-dir': 'NotebookApp.notebook_dir',
     'browser': 'NotebookApp.browser',
     'pylab': 'NotebookApp.pylab',
 })
 
 #-----------------------------------------------------------------------------
 # NotebookApp
 #-----------------------------------------------------------------------------
 
 class NotebookApp(BaseIPythonApplication):
 
     name = 'ipython-notebook'
     
     description = """
         The IPython HTML Notebook.
         
         This launches a Tornado based HTML Notebook Server that serves up an
         HTML5/Javascript Notebook client.
     """
     examples = _examples
     aliases = aliases
     flags = flags
     
     classes = [
         KernelManager, ProfileDir, Session, MappingKernelManager,
         ContentsManager, FileContentsManager, NotebookNotary,
     ]
     flags = Dict(flags)
     aliases = Dict(aliases)
     
     subcommands = dict(
         list=(NbserverListApp, NbserverListApp.description.splitlines()[0]),
     )
 
     ipython_kernel_argv = List(Unicode)
     
     _log_formatter_cls = LogFormatter
 
     def _log_level_default(self):
         return logging.INFO
 
     def _log_datefmt_default(self):
         """Exclude date from default date format"""
         return "%H:%M:%S"
     
     def _log_format_default(self):
         """override default log format to include time"""
         return u"%(color)s[%(levelname)1.1s %(asctime)s.%(msecs).03d %(name)s]%(end_color)s %(message)s"
 
     # create requested profiles by default, if they don't exist:
     auto_create = Bool(True)
 
     # file to be opened in the notebook server
     file_to_run = Unicode('', config=True)
 
     # Network related information
     
     allow_origin = Unicode('', config=True,
         help="""Set the Access-Control-Allow-Origin header
         
         Use '*' to allow any origin to access your server.
         
         Takes precedence over allow_origin_pat.
         """
     )
     
     allow_origin_pat = Unicode('', config=True,
         help="""Use a regular expression for the Access-Control-Allow-Origin header
         
         Requests from an origin matching the expression will get replies with:
         
             Access-Control-Allow-Origin: origin
         
         where `origin` is the origin of the request.
         
         Ignored if allow_origin is set.
         """
     )
     
     allow_credentials = Bool(False, config=True,
         help="Set the Access-Control-Allow-Credentials: true header"
     )
     
     default_url = Unicode('/tree', config=True,
         help="The default URL to redirect to from `/`"
     )
     
     ip = Unicode('localhost', config=True,
         help="The IP address the notebook server will listen on."
     )
 
     def _ip_changed(self, name, old, new):
         if new == u'*': self.ip = u''
 
     port = Integer(8888, config=True,
         help="The port the notebook server will listen on."
     )
     port_retries = Integer(50, config=True,
         help="The number of additional ports to try if the specified port is not available."
     )
 
     certfile = Unicode(u'', config=True, 
         help="""The full path to an SSL/TLS certificate file."""
     )
     
     keyfile = Unicode(u'', config=True, 
         help="""The full path to a private key file for usage with SSL/TLS."""
     )
     
     cookie_secret_file = Unicode(config=True,
         help="""The file where the cookie secret is stored."""
     )
     def _cookie_secret_file_default(self):
         if self.profile_dir is None:
             return ''
         return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret')
     
     cookie_secret = Bytes(b'', config=True,
         help="""The random bytes used to secure cookies.
         By default this is a new random number every time you start the Notebook.
         Set it to a value in a config file to enable logins to persist across server sessions.
         
         Note: Cookie secrets should be kept private, do not share config files with
         cookie_secret stored in plaintext (you can read the value from a file).
         """
     )
     def _cookie_secret_default(self):
         if os.path.exists(self.cookie_secret_file):
             with io.open(self.cookie_secret_file, 'rb') as f:
                 return f.read()
         else:
             secret = base64.encodestring(os.urandom(1024))
             self._write_cookie_secret_file(secret)
             return secret
     
     def _write_cookie_secret_file(self, secret):
         """write my secret to my secret_file"""
         self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file)
         with io.open(self.cookie_secret_file, 'wb') as f:
             f.write(secret)
         try:
             os.chmod(self.cookie_secret_file, 0o600)
         except OSError:
             self.log.warn(
                 "Could not set permissions on %s",
                 self.cookie_secret_file
             )
 
     password = Unicode(u'', config=True,
                       help="""Hashed password to use for web authentication.
 
                       To generate, type in a python/IPython shell:
 
                         from IPython.lib import passwd; passwd()
 
                       The string should be of the form type:salt:hashed-password.
                       """
     )
 
     open_browser = Bool(True, config=True,
                         help="""Whether to open in a browser after starting.
                         The specific browser used is platform dependent and
                         determined by the python standard library `webbrowser`
                         module, unless it is overridden using the --browser
                         (NotebookApp.browser) configuration option.
                         """)
 
     browser = Unicode(u'', config=True,
                       help="""Specify what command to use to invoke a web
                       browser when opening the notebook. If not specified, the
                       default browser will be determined by the `webbrowser`
                       standard library module, which allows setting of the
                       BROWSER environment variable to override it.
                       """)
     
     webapp_settings = Dict(config=True,
         help="DEPRECATED, use tornado_settings"
     )
     def _webapp_settings_changed(self, name, old, new):
         self.log.warn("\n    webapp_settings is deprecated, use tornado_settings.\n")
         self.tornado_settings = new
     
     tornado_settings = Dict(config=True,
             help="Supply overrides for the tornado.web.Application that the "
                  "IPython notebook uses.")
 
     jinja_environment_options = Dict(config=True, 
             help="Supply extra arguments that will be passed to Jinja environment.")
 
     
     enable_mathjax = Bool(True, config=True,
         help="""Whether to enable MathJax for typesetting math/TeX
 
         MathJax is the javascript library IPython uses to render math/LaTeX. It is
         very large, so you may want to disable it if you have a slow internet
         connection, or for offline use of the notebook.
 
         When disabled, equations etc. will appear as their untransformed TeX source.
         """
     )
     def _enable_mathjax_changed(self, name, old, new):
         """set mathjax url to empty if mathjax is disabled"""
         if not new:
             self.mathjax_url = u''
 
     base_url = Unicode('/', config=True,
                                help='''The base URL for the notebook server.
 
                                Leading and trailing slashes can be omitted,
                                and will automatically be added.
                                ''')
     def _base_url_changed(self, name, old, new):
         if not new.startswith('/'):
             self.base_url = '/'+new
         elif not new.endswith('/'):
             self.base_url = new+'/'
     
     base_project_url = Unicode('/', config=True, help="""DEPRECATED use base_url""")
     def _base_project_url_changed(self, name, old, new):
         self.log.warn("base_project_url is deprecated, use base_url")
         self.base_url = new
 
     extra_static_paths = List(Unicode, config=True,
         help="""Extra paths to search for serving static files.
         
         This allows adding javascript/css to be available from the notebook server machine,
         or overriding individual files in the IPython"""
     )
     def _extra_static_paths_default(self):
         return [os.path.join(self.profile_dir.location, 'static')]
     
     @property
     def static_file_path(self):
         """return extra paths + the default location"""
         return self.extra_static_paths + [DEFAULT_STATIC_FILES_PATH]
 
     extra_template_paths = List(Unicode, config=True,
         help="""Extra paths to search for serving jinja templates.
 
         Can be used to override templates from IPython.html.templates."""
     )
     def _extra_template_paths_default(self):
         return []
 
     @property
     def template_file_path(self):
         """return extra paths + the default locations"""
         return self.extra_template_paths + DEFAULT_TEMPLATE_PATH_LIST
 
     nbextensions_path = List(Unicode, config=True,
         help="""paths for Javascript extensions. By default, this is just IPYTHONDIR/nbextensions"""
     )
     def _nbextensions_path_default(self):
         return [os.path.join(get_ipython_dir(), 'nbextensions')]
 
     websocket_url = Unicode("", config=True,
         help="""The base URL for websockets,
         if it differs from the HTTP server (hint: it almost certainly doesn't).
         
         Should be in the form of an HTTP origin: ws[s]://hostname[:port]
         """
     )
     mathjax_url = Unicode("", config=True,
         help="""The url for MathJax.js."""
     )
     def _mathjax_url_default(self):
         if not self.enable_mathjax:
             return u''
         static_url_prefix = self.tornado_settings.get("static_url_prefix",
                          url_path_join(self.base_url, "static")
         )
         
         # try local mathjax, either in nbextensions/mathjax or static/mathjax
         for (url_prefix, search_path) in [
             (url_path_join(self.base_url, "nbextensions"), self.nbextensions_path),
             (static_url_prefix, self.static_file_path),
         ]:
             self.log.debug("searching for local mathjax in %s", search_path)
             try:
                 mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), search_path)
             except IOError:
                 continue
             else:
                 url = url_path_join(url_prefix, u"mathjax/MathJax.js")
                 self.log.info("Serving local MathJax from %s at %s", mathjax, url)
                 return url
         
         # no local mathjax, serve from CDN
         url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js"
         self.log.info("Using MathJax from CDN: %s", url)
         return url
     
     def _mathjax_url_changed(self, name, old, new):
         if new and not self.enable_mathjax:
             # enable_mathjax=False overrides mathjax_url
             self.mathjax_url = u''
         else:
             self.log.info("Using MathJax: %s", new)
 
     contents_manager_class = DottedObjectName('IPython.html.services.contents.filemanager.FileContentsManager',
         config=True,
         help='The notebook manager class to use.'
     )
     kernel_manager_class = DottedObjectName('IPython.html.services.kernels.kernelmanager.MappingKernelManager',
         config=True,
         help='The kernel manager class to use.'
     )
     session_manager_class = DottedObjectName('IPython.html.services.sessions.sessionmanager.SessionManager',
         config=True,
         help='The session manager class to use.'
     )
     cluster_manager_class = DottedObjectName('IPython.html.services.clusters.clustermanager.ClusterManager',
         config=True,
         help='The cluster manager class to use.'
     )
 
     config_manager_class = DottedObjectName('IPython.html.services.config.manager.ConfigManager',
         config = True,
         help='The config manager class to use'
     )
 
     kernel_spec_manager = Instance(KernelSpecManager)
 
     def _kernel_spec_manager_default(self):
         return KernelSpecManager(ipython_dir=self.ipython_dir)
 
 
     kernel_spec_manager_class = DottedObjectName('IPython.kernel.kernelspec.KernelSpecManager',
             config=True,
             help="""
             The kernel spec manager class to use. Should be a subclass
             of `IPython.kernel.kernelspec.KernelSpecManager`.
 
             The Api of KernelSpecManager is provisional and might change
             without warning between this version of IPython and the next stable one.
             """)
 
     trust_xheaders = Bool(False, config=True,
         help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
               "sent by the upstream reverse proxy. Necessary if the proxy handles SSL")
     )
 
     info_file = Unicode()
 
     def _info_file_default(self):
         info_file = "nbserver-%s.json"%os.getpid()
         return os.path.join(self.profile_dir.security_dir, info_file)
     
     pylab = Unicode('disabled', config=True,
         help="""
         DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.
         """
     )
     def _pylab_changed(self, name, old, new):
         """when --pylab is specified, display a warning and exit"""
         if new != 'warn':
             backend = ' %s' % new
         else:
             backend = ''
         self.log.error("Support for specifying --pylab on the command line has been removed.")
         self.log.error(
             "Please use `%pylab{0}` or `%matplotlib{0}` in the notebook itself.".format(backend)
         )
         self.exit(1)
 
     notebook_dir = Unicode(config=True,
         help="The directory to use for notebooks and kernels."
     )
 
     def _notebook_dir_default(self):
         if self.file_to_run:
             return os.path.dirname(os.path.abspath(self.file_to_run))
         else:
             return py3compat.getcwd()
 
     def _notebook_dir_changed(self, name, old, new):
         """Do a bit of validation of the notebook dir."""
         if not os.path.isabs(new):
             # If we receive a non-absolute path, make it absolute.
             self.notebook_dir = os.path.abspath(new)
             return
         if not os.path.isdir(new):
             raise TraitError("No such notebook dir: %r" % new)
         
         # setting App.notebook_dir implies setting notebook and kernel dirs as well
         self.config.FileContentsManager.root_dir = new
         self.config.MappingKernelManager.root_dir = new
         
 
     def parse_command_line(self, argv=None):
         super(NotebookApp, self).parse_command_line(argv)
         
         if self.extra_args:
             arg0 = self.extra_args[0]
             f = os.path.abspath(arg0)
             self.argv.remove(arg0)
             if not os.path.exists(f):
                 self.log.critical("No such file or directory: %s", f)
                 self.exit(1)
             
             # Use config here, to ensure that it takes higher priority than
             # anything that comes from the profile.
             c = Config()
             if os.path.isdir(f):
                 c.NotebookApp.notebook_dir = f
             elif os.path.isfile(f):
                 c.NotebookApp.file_to_run = f
             self.update_config(c)
 
     def init_kernel_argv(self):
         """add the profile-dir to arguments to be passed to IPython kernels"""
         # FIXME: remove special treatment of IPython kernels
         # Kernel should get *absolute* path to profile directory
         self.ipython_kernel_argv = ["--profile-dir", self.profile_dir.location]
 
     def init_configurables(self):
         # force Session default to be secure
         default_secure(self.config)
         kls = import_item(self.kernel_spec_manager_class)
         self.kernel_spec_manager = kls(ipython_dir=self.ipython_dir)
 
         kls = import_item(self.kernel_manager_class)
         self.kernel_manager = kls(
             parent=self, log=self.log, ipython_kernel_argv=self.ipython_kernel_argv,
             connection_dir = self.profile_dir.security_dir,
         )
         kls = import_item(self.contents_manager_class)
         self.contents_manager = kls(parent=self, log=self.log)
         kls = import_item(self.session_manager_class)
         self.session_manager = kls(parent=self, log=self.log,
                                    kernel_manager=self.kernel_manager,
                                    contents_manager=self.contents_manager)
         kls = import_item(self.cluster_manager_class)
         self.cluster_manager = kls(parent=self, log=self.log)
         self.cluster_manager.update_profiles()
 
         kls = import_item(self.config_manager_class)
         self.config_manager = kls(parent=self, log=self.log,
                                   profile_dir=self.profile_dir.location)
 
     def init_logging(self):
         # This prevents double log messages because tornado use a root logger that
         # self.log is a child of. The logging module dipatches log messages to a log
         # and all of its ancenstors until propagate is set to False.
         self.log.propagate = False
         
         for log in app_log, access_log, gen_log:
             # consistent log output name (NotebookApp instead of tornado.access, etc.)
             log.name = self.log.name
         # hook up tornado 3's loggers to our app handlers
         logger = logging.getLogger('tornado')
         logger.propagate = True
         logger.parent = self.log
         logger.setLevel(self.log.level)
     
     def init_webapp(self):
         """initialize tornado webapp and httpserver"""
         self.tornado_settings['allow_origin'] = self.allow_origin
         if self.allow_origin_pat:
             self.tornado_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat)
         self.tornado_settings['allow_credentials'] = self.allow_credentials
         
         self.web_app = NotebookWebApplication(
             self, self.kernel_manager, self.contents_manager,
             self.cluster_manager, self.session_manager, self.kernel_spec_manager,
             self.config_manager,
             self.log, self.base_url, self.default_url, self.tornado_settings,
             self.jinja_environment_options
         )
         if self.certfile:
             ssl_options = dict(certfile=self.certfile)
             if self.keyfile:
                 ssl_options['keyfile'] = self.keyfile
         else:
             ssl_options = None
         self.web_app.password = self.password
         self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options,
                                                  xheaders=self.trust_xheaders)
         if not self.ip:
             warning = "WARNING: The notebook server is listening on all IP addresses"
             if ssl_options is None:
                 self.log.critical(warning + " and not using encryption. This "
                     "is not recommended.")
             if not self.password:
                 self.log.critical(warning + " and not using authentication. "
                     "This is highly insecure and not recommended.")
         success = None
         for port in random_ports(self.port, self.port_retries+1):
             try:
                 self.http_server.listen(port, self.ip)
             except socket.error as e:
                 if e.errno == errno.EADDRINUSE:
                     self.log.info('The port %i is already in use, trying another random port.' % port)
                     continue
                 elif e.errno in (errno.EACCES, getattr(errno, 'WSAEACCES', errno.EACCES)):
                     self.log.warn("Permission to listen on port %i denied" % port)
                     continue
                 else:
                     raise
             else:
                 self.port = port
                 success = True
                 break
         if not success:
             self.log.critical('ERROR: the notebook server could not be started because '
                               'no available port could be found.')
             self.exit(1)
     
     @property
     def display_url(self):
         ip = self.ip if self.ip else '[all ip addresses on your system]'
         return self._url(ip)
 
     @property
     def connection_url(self):
         ip = self.ip if self.ip else 'localhost'
         return self._url(ip)
 
     def _url(self, ip):
         proto = 'https' if self.certfile else 'http'
         return "%s://%s:%i%s" % (proto, ip, self.port, self.base_url)
 
     def init_terminals(self):
         try:
             from .terminal import initialize
             initialize(self.web_app)
             self.web_app.settings['terminals_available'] = True
         except ImportError as e:
             self.log.info("Terminals not available (error was %s)", e)
 
     def init_signal(self):
         if not sys.platform.startswith('win'):
             signal.signal(signal.SIGINT, self._handle_sigint)
         signal.signal(signal.SIGTERM, self._signal_stop)
         if hasattr(signal, 'SIGUSR1'):
             # Windows doesn't support SIGUSR1
             signal.signal(signal.SIGUSR1, self._signal_info)
         if hasattr(signal, 'SIGINFO'):
             # only on BSD-based systems
             signal.signal(signal.SIGINFO, self._signal_info)
     
     def _handle_sigint(self, sig, frame):
         """SIGINT handler spawns confirmation dialog"""
         # register more forceful signal handler for ^C^C case
         signal.signal(signal.SIGINT, self._signal_stop)
         # request confirmation dialog in bg thread, to avoid
         # blocking the App
         thread = threading.Thread(target=self._confirm_exit)
         thread.daemon = True
         thread.start()
     
     def _restore_sigint_handler(self):
         """callback for restoring original SIGINT handler"""
         signal.signal(signal.SIGINT, self._handle_sigint)
     
     def _confirm_exit(self):
         """confirm shutdown on ^C
         
         A second ^C, or answering 'y' within 5s will cause shutdown,
         otherwise original SIGINT handler will be restored.
         
         This doesn't work on Windows.
         """
         info = self.log.info
         info('interrupted')
         print(self.notebook_info())
         sys.stdout.write("Shutdown this notebook server (y/[n])? ")
         sys.stdout.flush()
         r,w,x = select.select([sys.stdin], [], [], 5)
         if r:
             line = sys.stdin.readline()
             if line.lower().startswith('y') and 'n' not in line.lower():
                 self.log.critical("Shutdown confirmed")
                 ioloop.IOLoop.instance().stop()
                 return
         else:
             print("No answer for 5s:", end=' ')
         print("resuming operation...")
         # no answer, or answer is no:
         # set it back to original SIGINT handler
         # use IOLoop.add_callback because signal.signal must be called
         # from main thread
         ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
     
     def _signal_stop(self, sig, frame):
         self.log.critical("received signal %s, stopping", sig)
         ioloop.IOLoop.instance().stop()
 
     def _signal_info(self, sig, frame):
         print(self.notebook_info())
     
     def init_components(self):
         """Check the components submodule, and warn if it's unclean"""
         status = submodule.check_submodule_status()
         if status == 'missing':
             self.log.warn("components submodule missing, running `git submodule update`")
             submodule.update_submodules(submodule.ipython_parent())
         elif status == 'unclean':
             self.log.warn("components submodule unclean, you may see 404s on static/components")
             self.log.warn("run `setup.py submodule` or `git submodule update` to update")
     
     @catch_config_error
     def initialize(self, argv=None):
         super(NotebookApp, self).initialize(argv)
         self.init_logging()
         self.init_kernel_argv()
         self.init_configurables()
         self.init_components()
         self.init_webapp()
         self.init_terminals()
         self.init_signal()
 
     def cleanup_kernels(self):
         """Shutdown all kernels.
         
         The kernels will shutdown themselves when this process no longer exists,
         but explicit shutdown allows the KernelManagers to cleanup the connection files.
         """
         self.log.info('Shutting down kernels')
         self.kernel_manager.shutdown_all()
 
     def notebook_info(self):
         "Return the current working directory and the server url information"
         info = self.contents_manager.info_string() + "\n"
         info += "%d active kernels \n" % len(self.kernel_manager._kernels)
         return info + "The IPython Notebook is running at: %s" % self.display_url
 
     def server_info(self):
         """Return a JSONable dict of information about this server."""
         return {'url': self.connection_url,
                 'hostname': self.ip if self.ip else 'localhost',
                 'port': self.port,
                 'secure': bool(self.certfile),
                 'base_url': self.base_url,
                 'notebook_dir': os.path.abspath(self.notebook_dir),
                 'pid': os.getpid()
                }
 
     def write_server_info_file(self):
         """Write the result of server_info() to the JSON file info_file."""
         with open(self.info_file, 'w') as f:
             json.dump(self.server_info(), f, indent=2)
 
     def remove_server_info_file(self):
         """Remove the nbserver-<pid>.json file created for this server.
         
         Ignores the error raised when the file has already been removed.
         """
         try:
             os.unlink(self.info_file)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 raise
 
     def start(self):
         """ Start the IPython Notebook server app, after initialization
         
         This method takes no arguments so all configuration and initialization
         must be done prior to calling this method."""
         if self.subapp is not None:
             return self.subapp.start()
 
         info = self.log.info
         for line in self.notebook_info().split("\n"):
             info(line)
         info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")
 
         self.write_server_info_file()
 
         if self.open_browser or self.file_to_run:
             try:
                 browser = webbrowser.get(self.browser or None)
             except webbrowser.Error as e:
                 self.log.warn('No web browser found: %s.' % e)
                 browser = None
             
             if self.file_to_run:
                 if not os.path.exists(self.file_to_run):
                     self.log.critical("%s does not exist" % self.file_to_run)
                     self.exit(1)
 
                 relpath = os.path.relpath(self.file_to_run, self.notebook_dir)
                 uri = url_path_join('notebooks', *relpath.split(os.sep))
             else:
                 uri = 'tree'
             if browser:
                 b = lambda : browser.open(url_path_join(self.connection_url, uri),
                                           new=2)
                 threading.Thread(target=b).start()
         try:
             ioloop.IOLoop.instance().start()
         except KeyboardInterrupt:
             info("Interrupted...")
         finally:
             self.cleanup_kernels()
             self.remove_server_info_file()
 
 
 def list_running_servers(profile='default'):
     """Iterate over the server info files of running notebook servers.
     
     Given a profile name, find nbserver-* files in the security directory of
     that profile, and yield dicts of their information, each one pertaining to
     a currently running notebook server instance.
     """
     pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), name=profile)
     for file in os.listdir(pd.security_dir):
         if file.startswith('nbserver-'):
             with io.open(os.path.join(pd.security_dir, file), encoding='utf-8') as f:
                 info = json.load(f)
 
             # Simple check whether that process is really still running
             if check_pid(info['pid']):
                 yield info
             else:
                 # If the process has died, try to delete its info file
                 try:
                     os.unlink(file)
                 except OSError:
                     pass  # TODO: This should warn or log or something
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 launch_new_instance = NotebookApp.launch_instance
 

IPython/html/services/config/__init__.py

@@ -0,0 +1,1 b''
+from .manager import ConfigManager

IPython/html/services/kernels/tests/test_kernels_api.py

@@ -1,133 +1,139 b''
 """Test the kernels service API."""
 
 import json
 import requests
 
 from IPython.html.utils import url_path_join
 from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error
 
 class KernelAPI(object):
     """Wrapper for kernel REST API requests"""
     def __init__(self, base_url):
         self.base_url = base_url
 
     def _req(self, verb, path, body=None):
         response = requests.request(verb,
                 url_path_join(self.base_url, 'api/kernels', path), data=body)
 
         if 400 <= response.status_code < 600:
             try:
                 response.reason = response.json()['message']
             except:
                 pass
         response.raise_for_status()
 
         return response
 
     def list(self):
         return self._req('GET', '')
 
     def get(self, id):
         return self._req('GET', id)
 
     def start(self, name='python'):
         body = json.dumps({'name': name})
         return self._req('POST', '', body)
 
     def shutdown(self, id):
         return self._req('DELETE', id)
 
     def interrupt(self, id):
         return self._req('POST', url_path_join(id, 'interrupt'))
 
     def restart(self, id):
         return self._req('POST', url_path_join(id, 'restart'))
 
 class KernelAPITest(NotebookTestBase):
     """Test the kernels web service API"""
     def setUp(self):
         self.kern_api = KernelAPI(self.base_url())
 
     def tearDown(self):
         for k in self.kern_api.list().json():
             self.kern_api.shutdown(k['id'])
 
     def test__no_kernels(self):
         """Make sure there are no kernels running at the start"""
         kernels = self.kern_api.list().json()
         self.assertEqual(kernels, [])
 
     def test_default_kernel(self):
         # POST request
         r = self.kern_api._req('POST', '')
         kern1 = r.json()
         self.assertEqual(r.headers['location'], '/api/kernels/' + kern1['id'])
         self.assertEqual(r.status_code, 201)
         self.assertIsInstance(kern1, dict)
 
-        self.assertEqual(r.headers['x-frame-options'], "SAMEORIGIN")
+        self.assertEqual(r.headers['Content-Security-Policy'], (
+                            "frame-ancestors 'self'; "
+                            "report-uri /api/security/csp-report;"
+        ))
 
     def test_main_kernel_handler(self):
         # POST request
         r = self.kern_api.start()
         kern1 = r.json()
         self.assertEqual(r.headers['location'], '/api/kernels/' + kern1['id'])
         self.assertEqual(r.status_code, 201)
         self.assertIsInstance(kern1, dict)
 
-        self.assertEqual(r.headers['x-frame-options'], "SAMEORIGIN")
+        self.assertEqual(r.headers['Content-Security-Policy'], (
+                            "frame-ancestors 'self'; "
+                            "report-uri /api/security/csp-report;"
+        ))
 
         # GET request
         r = self.kern_api.list()
         self.assertEqual(r.status_code, 200)
         assert isinstance(r.json(), list)
         self.assertEqual(r.json()[0]['id'], kern1['id'])
         self.assertEqual(r.json()[0]['name'], kern1['name'])
 
         # create another kernel and check that they both are added to the
         # list of kernels from a GET request
         kern2 = self.kern_api.start().json()
         assert isinstance(kern2, dict)
         r = self.kern_api.list()
         kernels = r.json()
         self.assertEqual(r.status_code, 200)
         assert isinstance(kernels, list)
         self.assertEqual(len(kernels), 2)
 
         # Interrupt a kernel
         r = self.kern_api.interrupt(kern2['id'])
         self.assertEqual(r.status_code, 204)
 
         # Restart a kernel
         r = self.kern_api.restart(kern2['id'])
         self.assertEqual(r.headers['Location'], '/api/kernels/'+kern2['id'])
         rekern = r.json()
         self.assertEqual(rekern['id'], kern2['id'])
         self.assertEqual(rekern['name'], kern2['name'])
 
     def test_kernel_handler(self):
         # GET kernel with given id
         kid = self.kern_api.start().json()['id']
         r = self.kern_api.get(kid)
         kern1 = r.json()
         self.assertEqual(r.status_code, 200)
         assert isinstance(kern1, dict)
         self.assertIn('id', kern1)
         self.assertEqual(kern1['id'], kid)
 
         # Request a bad kernel id and check that a JSON
         # message is returned!
         bad_id = '111-111-111-111-111'
         with assert_http_error(404, 'Kernel does not exist: ' + bad_id):
             self.kern_api.get(bad_id)
 
         # DELETE kernel with id
         r = self.kern_api.shutdown(kid)
         self.assertEqual(r.status_code, 204)
         kernels = self.kern_api.list().json()
         self.assertEqual(kernels, [])
 
         # Request to delete a non-existent kernel id
         bad_id = '111-111-111-111-111'
         with assert_http_error(404, 'Kernel does not exist: ' + bad_id):
             self.kern_api.shutdown(bad_id)

IPython/html/static/services/kernels/kernel.js

@@ -1,1073 +1,1073 b''
 // Copyright (c) IPython Development Team.
 // Distributed under the terms of the Modified BSD License.
 
 define([
     'base/js/namespace',
     'jquery',
     'base/js/utils',
     './comm',
     './serialize',
     'widgets/js/init'
 ], function(IPython, $, utils, comm, serialize, widgetmanager) {
     "use strict";
 
     /**
      * A Kernel class to communicate with the Python kernel. This
      * should generally not be constructed directly, but be created
      * by.  the `Session` object. Once created, this object should be
      * used to communicate with the kernel.
      * 
      * @class Kernel
      * @param {string} kernel_service_url - the URL to access the kernel REST api
      * @param {string} ws_url - the websockets URL
      * @param {Notebook} notebook - notebook object
      * @param {string} name - the kernel type (e.g. python3)
      */
     var Kernel = function (kernel_service_url, ws_url, notebook, name) {
         this.events = notebook.events;
 
         this.id = null;
         this.name = name;
 
         this.channels = {
             'shell': null,
             'iopub': null,
             'stdin': null
         };
 
         this.kernel_service_url = kernel_service_url;
         this.kernel_url = null;
         this.ws_url = ws_url || IPython.utils.get_body_data("wsUrl");
         if (!this.ws_url) {
             // trailing 's' in https will become wss for secure web sockets
             this.ws_url = location.protocol.replace('http', 'ws') + "//" + location.host;
         }
 
         this.username = "username";
         this.session_id = utils.uuid();
         this._msg_callbacks = {};
         this.info_reply = {}; // kernel_info_reply stored here after starting
         this.unsolicited_msg_callback = null;
 
         if (typeof(WebSocket) !== 'undefined') {
             this.WebSocket = WebSocket;
         } else if (typeof(MozWebSocket) !== 'undefined') {
             this.WebSocket = MozWebSocket;
         } else {
             alert('Your browser does not have WebSocket support, please try Chrome, Safari or Firefox ≥ 6. Firefox 4 and 5 are also supported by you have to enable WebSockets in about:config.');
         }
         
         this.bind_events();
         this.init_iopub_handlers();
         this.comm_manager = new comm.CommManager(this);
         this.widget_manager = new widgetmanager.WidgetManager(this.comm_manager, notebook);
         
         this.last_msg_id = null;
         this.last_msg_callbacks = {};
 
         this._autorestart_attempt = 0;
         this._reconnect_attempt = 0;
         this.reconnect_limit = 7;
     };
 
     /**
      * @function _get_msg
      */
     Kernel.prototype._get_msg = function (msg_type, content, metadata, buffers) {
         var msg = {
             header : {
                 msg_id : utils.uuid(),
                 username : this.username,
                 session : this.session_id,
                 msg_type : msg_type,
                 version : "5.0"
             },
             metadata : metadata || {},
             content : content,
             buffers : buffers || [],
             parent_header : {}
         };
         return msg;
     };
 
     /**
      * @function bind_events
      */
     Kernel.prototype.bind_events = function () {
         var that = this;
         this.events.on('send_input_reply.Kernel', function(evt, data) { 
             that.send_input_reply(data);
         });
 
         var record_status = function (evt, info) {
             console.log('Kernel: ' + evt.type + ' (' + info.kernel.id + ')');
         };
 
         this.events.on('kernel_created.Kernel', record_status);
         this.events.on('kernel_reconnecting.Kernel', record_status);
         this.events.on('kernel_connected.Kernel', record_status);
         this.events.on('kernel_starting.Kernel', record_status);
         this.events.on('kernel_restarting.Kernel', record_status);
         this.events.on('kernel_autorestarting.Kernel', record_status);
         this.events.on('kernel_interrupting.Kernel', record_status);
         this.events.on('kernel_disconnected.Kernel', record_status);
         // these are commented out because they are triggered a lot, but can
         // be uncommented for debugging purposes
         //this.events.on('kernel_idle.Kernel', record_status);
         //this.events.on('kernel_busy.Kernel', record_status);
         this.events.on('kernel_ready.Kernel', record_status);
         this.events.on('kernel_killed.Kernel', record_status);
         this.events.on('kernel_dead.Kernel', record_status);
 
         this.events.on('kernel_ready.Kernel', function () {
             that._autorestart_attempt = 0;
         });
         this.events.on('kernel_connected.Kernel', function () {
             that._reconnect_attempt = 0;
         });
     };
 
     /**
      * Initialize the iopub handlers.
      *
      * @function init_iopub_handlers
      */
     Kernel.prototype.init_iopub_handlers = function () {
         var output_msg_types = ['stream', 'display_data', 'execute_result', 'error'];
         this._iopub_handlers = {};
         this.register_iopub_handler('status', $.proxy(this._handle_status_message, this));
         this.register_iopub_handler('clear_output', $.proxy(this._handle_clear_output, this));
         this.register_iopub_handler('execute_input', $.proxy(this._handle_input_message, this));
         
         for (var i=0; i < output_msg_types.length; i++) {
             this.register_iopub_handler(output_msg_types[i], $.proxy(this._handle_output_message, this));
         }
     };
 
     /**
      * GET /api/kernels
      *
      * Get the list of running kernels.
      *
      * @function list
      * @param {function} [success] - function executed on ajax success
      * @param {function} [error] - functon executed on ajax error
      */
     Kernel.prototype.list = function (success, error) {
         $.ajax(this.kernel_service_url, {
             processData: false,
             cache: false,
             type: "GET",
             dataType: "json",
             success: success,
             error: this._on_error(error)
         });
     };
 
     /**
      * POST /api/kernels
      *
      * Start a new kernel.
      *
      * In general this shouldn't be used -- the kernel should be
      * started through the session API. If you use this function and
      * are also using the session API then your session and kernel
      * WILL be out of sync!
      *
      * @function start
      * @param {params} [Object] - parameters to include in the query string
      * @param {function} [success] - function executed on ajax success
      * @param {function} [error] - functon executed on ajax error
      */
     Kernel.prototype.start = function (params, success, error) {
         var url = this.kernel_service_url;
         var qs = $.param(params || {}); // query string for sage math stuff
         if (qs !== "") {
             url = url + "?" + qs;
         }
 
         var that = this;
         var on_success = function (data, status, xhr) {
             that.events.trigger('kernel_created.Kernel', {kernel: that});
             that._kernel_created(data);
             if (success) {
                 success(data, status, xhr);
             }
         };
 
         $.ajax(url, {
             processData: false,
             cache: false,
             type: "POST",
             data: JSON.stringify({name: this.name}),
             dataType: "json",
             success: this._on_success(on_success),
             error: this._on_error(error)
         });
 
         return url;
     };
 
     /**
      * GET /api/kernels/[:kernel_id]
      *
      * Get information about the kernel.
      *
      * @function get_info
      * @param {function} [success] - function executed on ajax success
      * @param {function} [error] - functon executed on ajax error
      */
     Kernel.prototype.get_info = function (success, error) {
         $.ajax(this.kernel_url, {
             processData: false,
             cache: false,
             type: "GET",
             dataType: "json",
             success: this._on_success(success),
             error: this._on_error(error)
         });
     };
 
     /**
      * DELETE /api/kernels/[:kernel_id]
      *
      * Shutdown the kernel.
      *
      * If you are also using sessions, then this function shoul NOT be
      * used. Instead, use Session.delete. Otherwise, the session and
      * kernel WILL be out of sync.
      *
      * @function kill
      * @param {function} [success] - function executed on ajax success
      * @param {function} [error] - functon executed on ajax error
      */
     Kernel.prototype.kill = function (success, error) {
         this.events.trigger('kernel_killed.Kernel', {kernel: this});
         this._kernel_dead();
         $.ajax(this.kernel_url, {
             processData: false,
             cache: false,
             type: "DELETE",
             dataType: "json",
             success: this._on_success(success),
             error: this._on_error(error)
         });
     };
 
     /**
      * POST /api/kernels/[:kernel_id]/interrupt
      *
      * Interrupt the kernel.
      *
      * @function interrupt
      * @param {function} [success] - function executed on ajax success
      * @param {function} [error] - functon executed on ajax error
      */
     Kernel.prototype.interrupt = function (success, error) {
         this.events.trigger('kernel_interrupting.Kernel', {kernel: this});
 
         var that = this;
         var on_success = function (data, status, xhr) {
             // get kernel info so we know what state the kernel is in
             that.kernel_info();
             if (success) {
                 success(data, status, xhr);
             }
         };
 
         var url = utils.url_join_encode(this.kernel_url, 'interrupt');
         $.ajax(url, {
             processData: false,
             cache: false,
             type: "POST",
             dataType: "json",
             success: this._on_success(on_success),
             error: this._on_error(error)
         });
     };
 
+    Kernel.prototype.restart = function (success, error) {
     /**
      * POST /api/kernels/[:kernel_id]/restart
      *
      * Restart the kernel.
      *
      * @function interrupt
      * @param {function} [success] - function executed on ajax success
      * @param {function} [error] - functon executed on ajax error
      */
-    Kernel.prototype.restart = function (success, error) {
         this.events.trigger('kernel_restarting.Kernel', {kernel: this});
         this.stop_channels();
 
         var that = this;
         var on_success = function (data, status, xhr) {
             that.events.trigger('kernel_created.Kernel', {kernel: that});
             that._kernel_created(data);
             if (success) {
                 success(data, status, xhr);
             }
         };
 
         var on_error = function (xhr, status, err) {
             that.events.trigger('kernel_dead.Kernel', {kernel: that});
             that._kernel_dead();
             if (error) {
                 error(xhr, status, err);
             }
         };
 
         var url = utils.url_join_encode(this.kernel_url, 'restart');
         $.ajax(url, {
             processData: false,
             cache: false,
             type: "POST",
             dataType: "json",
             success: this._on_success(on_success),
             error: this._on_error(on_error)
         });
     };
 
+    Kernel.prototype.reconnect = function () {
     /**
      * Reconnect to a disconnected kernel. This is not actually a
      * standard HTTP request, but useful function nonetheless for
      * reconnecting to the kernel if the connection is somehow lost.
      *
      * @function reconnect
      */
-    Kernel.prototype.reconnect = function () {
         if (this.is_connected()) {
             return;
         }
         this._reconnect_attempt = this._reconnect_attempt + 1;
         this.events.trigger('kernel_reconnecting.Kernel', {
             kernel: this,
             attempt: this._reconnect_attempt,
         });
         this.start_channels();
     };
 
+    Kernel.prototype._on_success = function (success) {
     /**
      * Handle a successful AJAX request by updating the kernel id and
      * name from the response, and then optionally calling a provided
      * callback.
      *
      * @function _on_success
      * @param {function} success - callback
      */
-    Kernel.prototype._on_success = function (success) {
         var that = this;
         return function (data, status, xhr) {
             if (data) {
                 that.id = data.id;
                 that.name = data.name;
             }
             that.kernel_url = utils.url_join_encode(that.kernel_service_url, that.id);
             if (success) {
                 success(data, status, xhr);
             }
         };
     };
 
+    Kernel.prototype._on_error = function (error) {
     /**
      * Handle a failed AJAX request by logging the error message, and
      * then optionally calling a provided callback.
      *
      * @function _on_error
      * @param {function} error - callback
      */
-    Kernel.prototype._on_error = function (error) {
         return function (xhr, status, err) {
             utils.log_ajax_error(xhr, status, err);
             if (error) {
                 error(xhr, status, err);
             }
         };
     };
 
+    Kernel.prototype._kernel_created = function (data) {
     /**
      * Perform necessary tasks once the kernel has been started,
      * including actually connecting to the kernel.
      *
      * @function _kernel_created
      * @param {Object} data - information about the kernel including id
      */
-    Kernel.prototype._kernel_created = function (data) {
         this.id = data.id;
         this.kernel_url = utils.url_join_encode(this.kernel_service_url, this.id);
         this.start_channels();
     };
 
+    Kernel.prototype._kernel_connected = function () {
     /**
      * Perform necessary tasks once the connection to the kernel has
      * been established. This includes requesting information about
      * the kernel.
      *
      * @function _kernel_connected
      */
-    Kernel.prototype._kernel_connected = function () {
         this.events.trigger('kernel_connected.Kernel', {kernel: this});
         this.events.trigger('kernel_starting.Kernel', {kernel: this});
         // get kernel info so we know what state the kernel is in
         var that = this;
         this.kernel_info(function (reply) {
             that.info_reply = reply.content;
             that.events.trigger('kernel_ready.Kernel', {kernel: that});
         });
     };
 
+    Kernel.prototype._kernel_dead = function () {
     /**
      * Perform necessary tasks after the kernel has died. This closing
      * communication channels to the kernel if they are still somehow
      * open.
      *
      * @function _kernel_dead
      */
-    Kernel.prototype._kernel_dead = function () {
         this.stop_channels();
     };
 
+    Kernel.prototype.start_channels = function () {
     /**
      * Start the `shell`and `iopub` channels.
      * Will stop and restart them if they already exist.
      *
      * @function start_channels
      */
-    Kernel.prototype.start_channels = function () {
         var that = this;
         this.stop_channels();
         var ws_host_url = this.ws_url + this.kernel_url;
 
         console.log("Starting WebSockets:", ws_host_url);
         
         var channel_url = function(channel) {
             return [
                 that.ws_url,
                 utils.url_join_encode(that.kernel_url, channel),
                 "?session_id=" + that.session_id
             ].join('');
         };
         this.channels.shell = new this.WebSocket(channel_url("shell"));
         this.channels.stdin = new this.WebSocket(channel_url("stdin"));
         this.channels.iopub = new this.WebSocket(channel_url("iopub"));
         
         var already_called_onclose = false; // only alert once
         var ws_closed_early = function(evt){
             if (already_called_onclose){
                 return;
             }
             already_called_onclose = true;
             if ( ! evt.wasClean ){
                 // If the websocket was closed early, that could mean
                 // that the kernel is actually dead. Try getting
                 // information about the kernel from the API call --
                 // if that fails, then assume the kernel is dead,
                 // otherwise just follow the typical websocket closed
                 // protocol.
                 that.get_info(function () {
                     that._ws_closed(ws_host_url, false);
                 }, function () {
                     that.events.trigger('kernel_dead.Kernel', {kernel: that});
                     that._kernel_dead();
                 });
             }
         };
         var ws_closed_late = function(evt){
             if (already_called_onclose){
                 return;
             }
             already_called_onclose = true;
             if ( ! evt.wasClean ){
                 that._ws_closed(ws_host_url, false);
             }
         };
         var ws_error = function(evt){
             if (already_called_onclose){
                 return;
             }
             already_called_onclose = true;
             that._ws_closed(ws_host_url, true);
         };
 
         for (var c in this.channels) {
             this.channels[c].onopen = $.proxy(this._ws_opened, this);
             this.channels[c].onclose = ws_closed_early;
             this.channels[c].onerror = ws_error;
         }
         // switch from early-close to late-close message after 1s
         setTimeout(function() {
             for (var c in that.channels) {
                 if (that.channels[c] !== null) {
                     that.channels[c].onclose = ws_closed_late;
                 }
             }
         }, 1000);
         this.channels.shell.onmessage = $.proxy(this._handle_shell_reply, this);
         this.channels.iopub.onmessage = $.proxy(this._handle_iopub_message, this);
         this.channels.stdin.onmessage = $.proxy(this._handle_input_request, this);
     };
 
+    Kernel.prototype._ws_opened = function (evt) {
     /**
      * Handle a websocket entering the open state,
      * signaling that the kernel is connected when all channels are open.
      *
      * @function _ws_opened
      */
-    Kernel.prototype._ws_opened = function (evt) {
         if (this.is_connected()) {
             // all events ready, trigger started event.
             this._kernel_connected();
         }
     };
 
+    Kernel.prototype._ws_closed = function(ws_url, error) {
     /**
      * Handle a websocket entering the closed state. This closes the
      * other communication channels if they are open. If the websocket
      * was not closed due to an error, try to reconnect to the kernel.
      *
      * @function _ws_closed
      * @param {string} ws_url - the websocket url
      * @param {bool} error - whether the connection was closed due to an error
      */
-    Kernel.prototype._ws_closed = function(ws_url, error) {
         this.stop_channels();
 
         this.events.trigger('kernel_disconnected.Kernel', {kernel: this});
         if (error) {
             console.log('WebSocket connection failed: ', ws_url);
             this.events.trigger('kernel_connection_failed.Kernel', {kernel: this, ws_url: ws_url, attempt: this._reconnect_attempt});
         }
         this._schedule_reconnect();
     };
     
     Kernel.prototype._schedule_reconnect = function () {
         // function to call when kernel connection is lost
         // schedules reconnect, or fires 'connection_dead' if reconnect limit is hit
         if (this._reconnect_attempt < this.reconnect_limit) {
             var timeout = Math.pow(2, this._reconnect_attempt);
             console.log("Connection lost, reconnecting in " + timeout + " seconds.");
             setTimeout($.proxy(this.reconnect, this), 1e3 * timeout);
         } else {
             this.events.trigger('kernel_connection_dead.Kernel', {
                 kernel: this,
                 reconnect_attempt: this._reconnect_attempt,
             });
             console.log("Failed to reconnect, giving up.");
         }
     };
     
+    Kernel.prototype.stop_channels = function () {
     /**
      * Close the websocket channels. After successful close, the value
      * in `this.channels[channel_name]` will be null.
      *
      * @function stop_channels
      */
-    Kernel.prototype.stop_channels = function () {
         var that = this;
         var close = function (c) {
             return function () {
                 if (that.channels[c] && that.channels[c].readyState === WebSocket.CLOSED) {
                     that.channels[c] = null;
                 }
             };
         };
         for (var c in this.channels) {
             if ( this.channels[c] !== null ) {
                 if (this.channels[c].readyState === WebSocket.OPEN) {
                     this.channels[c].onclose = close(c);
                     this.channels[c].close();
                 } else {
                     close(c)();
                 }
             }
         }
     };
 
+    Kernel.prototype.is_connected = function () {
     /**
      * Check whether there is a connection to the kernel. This
      * function only returns true if all channel objects have been
      * created and have a state of WebSocket.OPEN.
      *
      * @function is_connected
      * @returns {bool} - whether there is a connection
      */
-    Kernel.prototype.is_connected = function () {
         for (var c in this.channels) {
             // if any channel is not ready, then we're not connected
             if (this.channels[c] === null) {
                 return false;
             }
             if (this.channels[c].readyState !== WebSocket.OPEN) {
                 return false;
             }
         }
         return true;
     };
 
+    Kernel.prototype.is_fully_disconnected = function () {
     /**
      * Check whether the connection to the kernel has been completely
      * severed. This function only returns true if all channel objects
      * are null.
      *
      * @function is_fully_disconnected
      * @returns {bool} - whether the kernel is fully disconnected
      */
-    Kernel.prototype.is_fully_disconnected = function () {
         for (var c in this.channels) {
             if (this.channels[c] === null) {
                 return true;
             }
         }
         return false;
     };
     
+    Kernel.prototype.send_shell_message = function (msg_type, content, callbacks, metadata, buffers) {
     /**
      * Send a message on the Kernel's shell channel
      *
      * @function send_shell_message
      */
-    Kernel.prototype.send_shell_message = function (msg_type, content, callbacks, metadata, buffers) {
         if (!this.is_connected()) {
             throw new Error("kernel is not connected");
         }
         var msg = this._get_msg(msg_type, content, metadata, buffers);
         this.channels.shell.send(serialize.serialize(msg));
         this.set_callbacks_for_msg(msg.header.msg_id, callbacks);
         return msg.header.msg_id;
     };
 
+    Kernel.prototype.kernel_info = function (callback) {
     /**
      * Get kernel info
      *
      * @function kernel_info
      * @param callback {function}
      *
      * When calling this method, pass a callback function that expects one argument.
      * The callback will be passed the complete `kernel_info_reply` message documented
      * [here](http://ipython.org/ipython-doc/dev/development/messaging.html#kernel-info)
      */
-    Kernel.prototype.kernel_info = function (callback) {
         var callbacks;
         if (callback) {
             callbacks = { shell : { reply : callback } };
         }
         return this.send_shell_message("kernel_info_request", {}, callbacks);
     };
 
+    Kernel.prototype.inspect = function (code, cursor_pos, callback) {
     /**
      * Get info on an object
      *
      * When calling this method, pass a callback function that expects one argument.
      * The callback will be passed the complete `inspect_reply` message documented
      * [here](http://ipython.org/ipython-doc/dev/development/messaging.html#object-information)
      *
      * @function inspect
      * @param code {string}
      * @param cursor_pos {integer}
      * @param callback {function}
      */
-    Kernel.prototype.inspect = function (code, cursor_pos, callback) {
         var callbacks;
         if (callback) {
             callbacks = { shell : { reply : callback } };
         }
         
         var content = {
             code : code,
             cursor_pos : cursor_pos,
             detail_level : 0
         };
         return this.send_shell_message("inspect_request", content, callbacks);
     };
 
+    Kernel.prototype.execute = function (code, callbacks, options) {
     /**
      * Execute given code into kernel, and pass result to callback.
      *
      * @async
      * @function execute
      * @param {string} code
      * @param [callbacks] {Object} With the following keys (all optional)
      *      @param callbacks.shell.reply {function}
      *      @param callbacks.shell.payload.[payload_name] {function}
      *      @param callbacks.iopub.output {function}
      *      @param callbacks.iopub.clear_output {function}
      *      @param callbacks.input {function}
      * @param {object} [options]
      *      @param [options.silent=false] {Boolean}
      *      @param [options.user_expressions=empty_dict] {Dict}
      *      @param [options.allow_stdin=false] {Boolean} true|false
      *
      * @example
      *
      * The options object should contain the options for the execute
      * call. Its default values are:
      *
      *      options = {
      *        silent : true,
      *        user_expressions : {},
      *        allow_stdin : false
      *      }
      *
      * When calling this method pass a callbacks structure of the
      * form:
      *
      *      callbacks = {
      *       shell : {
      *         reply : execute_reply_callback,
      *         payload : {
      *           set_next_input : set_next_input_callback,
      *         }
      *       },
      *       iopub : {
      *         output : output_callback,
      *         clear_output : clear_output_callback,
      *       },
      *       input : raw_input_callback
      *      }
      *
      * Each callback will be passed the entire message as a single
      * arugment.  Payload handlers will be passed the corresponding
      * payload and the execute_reply message.
      */
-    Kernel.prototype.execute = function (code, callbacks, options) {
         var content = {
             code : code,
             silent : true,
             store_history : false,
             user_expressions : {},
             allow_stdin : false
         };
         callbacks = callbacks || {};
         if (callbacks.input !== undefined) {
             content.allow_stdin = true;
         }
         $.extend(true, content, options);
         this.events.trigger('execution_request.Kernel', {kernel: this, content: content});
         return this.send_shell_message("execute_request", content, callbacks);
     };
 
     /**
      * When calling this method, pass a function to be called with the
      * `complete_reply` message as its only argument when it arrives.
      *
      * `complete_reply` is documented
      * [here](http://ipython.org/ipython-doc/dev/development/messaging.html#complete)
      *
      * @function complete
      * @param code {string}
      * @param cursor_pos {integer}
      * @param callback {function}
      */
     Kernel.prototype.complete = function (code, cursor_pos, callback) {
         var callbacks;
         if (callback) {
             callbacks = { shell : { reply : callback } };
         }
         var content = {
             code : code,
             cursor_pos : cursor_pos
         };
         return this.send_shell_message("complete_request", content, callbacks);
     };
 
     /**
      * @function send_input_reply
      */
     Kernel.prototype.send_input_reply = function (input) {
         if (!this.is_connected()) {
             throw new Error("kernel is not connected");
         }
         var content = {
             value : input
         };
         this.events.trigger('input_reply.Kernel', {kernel: this, content: content});
         var msg = this._get_msg("input_reply", content);
         this.channels.stdin.send(serialize.serialize(msg));
         return msg.header.msg_id;
     };
 
     /**
      * @function register_iopub_handler
      */
     Kernel.prototype.register_iopub_handler = function (msg_type, callback) {
         this._iopub_handlers[msg_type] = callback;
     };
 
     /**
      * Get the iopub handler for a specific message type.
      *
      * @function get_iopub_handler
      */
     Kernel.prototype.get_iopub_handler = function (msg_type) {
         return this._iopub_handlers[msg_type];
     };
 
     /**
      * Get callbacks for a specific message.
      *
      * @function get_callbacks_for_msg
      */
     Kernel.prototype.get_callbacks_for_msg = function (msg_id) {
         if (msg_id == this.last_msg_id) {
             return this.last_msg_callbacks;
         } else {
             return this._msg_callbacks[msg_id];
         }
     };
 
     /**
      * Clear callbacks for a specific message.
      *
      * @function clear_callbacks_for_msg
      */
     Kernel.prototype.clear_callbacks_for_msg = function (msg_id) {
         if (this._msg_callbacks[msg_id] !== undefined ) {
             delete this._msg_callbacks[msg_id];
         }
     };
     
     /**
      * @function _finish_shell
      */
     Kernel.prototype._finish_shell = function (msg_id) {
         var callbacks = this._msg_callbacks[msg_id];
         if (callbacks !== undefined) {
             callbacks.shell_done = true;
             if (callbacks.iopub_done) {
                 this.clear_callbacks_for_msg(msg_id);
             }
         }
     };
 
     /**
      * @function _finish_iopub
      */
     Kernel.prototype._finish_iopub = function (msg_id) {
         var callbacks = this._msg_callbacks[msg_id];
         if (callbacks !== undefined) {
             callbacks.iopub_done = true;
             if (callbacks.shell_done) {
                 this.clear_callbacks_for_msg(msg_id);
             }
         }
     };
     
     /**
      * Set callbacks for a particular message.
      * Callbacks should be a struct of the following form:
      * shell : {
      * 
      * }
      *
      * @function set_callbacks_for_msg
      */
     Kernel.prototype.set_callbacks_for_msg = function (msg_id, callbacks) {
         this.last_msg_id = msg_id;
         if (callbacks) {
             // shallow-copy mapping, because we will modify it at the top level
             var cbcopy = this._msg_callbacks[msg_id] = this.last_msg_callbacks = {};
             cbcopy.shell = callbacks.shell;
             cbcopy.iopub = callbacks.iopub;
             cbcopy.input = callbacks.input;
             cbcopy.shell_done = (!callbacks.shell);
             cbcopy.iopub_done = (!callbacks.iopub);
         } else {
             this.last_msg_callbacks = {};
         }
     };
 
     /**
      * @function _handle_shell_reply
      */
     Kernel.prototype._handle_shell_reply = function (e) {
         serialize.deserialize(e.data, $.proxy(this._finish_shell_reply, this));
     };
 
     Kernel.prototype._finish_shell_reply = function (reply) {
         this.events.trigger('shell_reply.Kernel', {kernel: this, reply:reply});
         var content = reply.content;
         var metadata = reply.metadata;
         var parent_id = reply.parent_header.msg_id;
         var callbacks = this.get_callbacks_for_msg(parent_id);
         if (!callbacks || !callbacks.shell) {
             return;
         }
         var shell_callbacks = callbacks.shell;
         
         // signal that shell callbacks are done
         this._finish_shell(parent_id);
         
         if (shell_callbacks.reply !== undefined) {
             shell_callbacks.reply(reply);
         }
         if (content.payload && shell_callbacks.payload) {
             this._handle_payloads(content.payload, shell_callbacks.payload, reply);
         }
     };
 
     /**
      * @function _handle_payloads
      */
     Kernel.prototype._handle_payloads = function (payloads, payload_callbacks, msg) {
         var l = payloads.length;
         // Payloads are handled by triggering events because we don't want the Kernel
         // to depend on the Notebook or Pager classes.
         for (var i=0; i<l; i++) {
             var payload = payloads[i];
             var callback = payload_callbacks[payload.source];
             if (callback) {
                 callback(payload, msg);
             }
         }
     };
 
     /**
      * @function _handle_status_message
      */
     Kernel.prototype._handle_status_message = function (msg) {
         var execution_state = msg.content.execution_state;
         var parent_id = msg.parent_header.msg_id;
         
         // dispatch status msg callbacks, if any
         var callbacks = this.get_callbacks_for_msg(parent_id);
         if (callbacks && callbacks.iopub && callbacks.iopub.status) {
             try {
                 callbacks.iopub.status(msg);
             } catch (e) {
                 console.log("Exception in status msg handler", e, e.stack);
             }
         }
         
         if (execution_state === 'busy') {
             this.events.trigger('kernel_busy.Kernel', {kernel: this});
 
         } else if (execution_state === 'idle') {
             // signal that iopub callbacks are (probably) done
             // async output may still arrive,
             // but only for the most recent request
             this._finish_iopub(parent_id);
             
             // trigger status_idle event
             this.events.trigger('kernel_idle.Kernel', {kernel: this});
 
         } else if (execution_state === 'starting') {
             this.events.trigger('kernel_starting.Kernel', {kernel: this});
             var that = this;
             this.kernel_info(function (reply) {
                 that.info_reply = reply.content;
                 that.events.trigger('kernel_ready.Kernel', {kernel: that});
             });
 
         } else if (execution_state === 'restarting') {
             // autorestarting is distinct from restarting,
             // in that it means the kernel died and the server is restarting it.
             // kernel_restarting sets the notification widget,
             // autorestart shows the more prominent dialog.
             this._autorestart_attempt = this._autorestart_attempt + 1;
             this.events.trigger('kernel_restarting.Kernel', {kernel: this});
             this.events.trigger('kernel_autorestarting.Kernel', {kernel: this, attempt: this._autorestart_attempt});
 
         } else if (execution_state === 'dead') {
             this.events.trigger('kernel_dead.Kernel', {kernel: this});
             this._kernel_dead();
         }
     };
     
     /**
      * Handle clear_output message
      *
      * @function _handle_clear_output
      */
     Kernel.prototype._handle_clear_output = function (msg) {
         var callbacks = this.get_callbacks_for_msg(msg.parent_header.msg_id);
         if (!callbacks || !callbacks.iopub) {
             return;
         }
         var callback = callbacks.iopub.clear_output;
         if (callback) {
             callback(msg);
         }
     };
 
     /**
      * handle an output message (execute_result, display_data, etc.)
      *
      * @function _handle_output_message
      */
     Kernel.prototype._handle_output_message = function (msg) {
         var callbacks = this.get_callbacks_for_msg(msg.parent_header.msg_id);
         if (!callbacks || !callbacks.iopub) {
             if (this.unsolicited_msg_callback) {
                 // The message came from another client. Let the UI decide what
                 // to do with it.
                 this.unsolicited_msg_callback(msg);
             }
             return;
         }
         var callback = callbacks.iopub.output;
         if (callback) {
             callback(msg);
         }
     };
 
     /**
      * Handle an input message (execute_input).
      *
      * @function _handle_input message
      */
     Kernel.prototype._handle_input_message = function (msg) {
         var callbacks = this.get_callbacks_for_msg(msg.parent_header.msg_id);
         if (!callbacks && this.unsolicited_msg_callback) {
             // The message came from another client. Let the UI decide what to
             // do with it.
             this.unsolicited_msg_callback(msg);
         }
     };
 
     /**
      * Dispatch IOPub messages to respective handlers. Each message
      * type should have a handler.
      *
      * @function _handle_iopub_message
      */
     Kernel.prototype._handle_iopub_message = function (e) {
         serialize.deserialize(e.data, $.proxy(this._finish_iopub_message, this));
     };
 
 
     Kernel.prototype._finish_iopub_message = function (msg) {
         var handler = this.get_iopub_handler(msg.header.msg_type);
         if (handler !== undefined) {
             handler(msg);
         }
     };
 
     /**
      * @function _handle_input_request
      */
     Kernel.prototype._handle_input_request = function (e) {
         serialize.deserialize(e.data, $.proxy(this._finish_input_request, this));
     };
 
 
     Kernel.prototype._finish_input_request = function (request) {
         var header = request.header;
         var content = request.content;
         var metadata = request.metadata;
         var msg_type = header.msg_type;
         if (msg_type !== 'input_request') {
             console.log("Invalid input request!", request);
             return;
         }
         var callbacks = this.get_callbacks_for_msg(request.parent_header.msg_id);
         if (callbacks) {
             if (callbacks.input) {
                 callbacks.input(request);
             }
         }
     };
 
     // Backwards compatability.
     IPython.Kernel = Kernel;
 
     return {'Kernel': Kernel};
 });

IPython/html/static/widgets/js/manager.js

@@ -1,240 +1,239 b''
 // Copyright (c) IPython Development Team.
 // Distributed under the terms of the Modified BSD License.
 
 define([
     "underscore",
     "backbone",
     "jquery",
     "base/js/utils",
     "base/js/namespace",
 ], function (_, Backbone, $, utils, IPython) {
     "use strict";
     //--------------------------------------------------------------------
     // WidgetManager class
     //--------------------------------------------------------------------
     var WidgetManager = function (comm_manager, notebook) {
         // Public constructor
         WidgetManager._managers.push(this);
 
         // Attach a comm manager to the 
         this.keyboard_manager = notebook.keyboard_manager;
         this.notebook = notebook;
         this.comm_manager = comm_manager;
         this._models = {}; /* Dictionary of model ids and model instances */
 
         // Register with the comm manager.
         this.comm_manager.register_target('ipython.widget', $.proxy(this._handle_comm_open, this));
     };
 
     //--------------------------------------------------------------------
     // Class level
     //--------------------------------------------------------------------
     WidgetManager._model_types = {}; /* Dictionary of model type names (target_name) and model types. */
     WidgetManager._view_types = {}; /* Dictionary of view names and view types. */
     WidgetManager._managers = []; /* List of widget managers */
 
     WidgetManager.register_widget_model = function (model_name, model_type) {
         // Registers a widget model by name.
         WidgetManager._model_types[model_name] = model_type;
     };
 
     WidgetManager.register_widget_view = function (view_name, view_type) {
         // Registers a widget view by name.
         WidgetManager._view_types[view_name] = view_type;
     };
 
     //--------------------------------------------------------------------
     // Instance level
     //--------------------------------------------------------------------
     WidgetManager.prototype.display_view = function(msg, model) {
         // Displays a view for a particular model.
         var that = this;
         var cell = this.get_msg_cell(msg.parent_header.msg_id);
         if (cell === null) {
             return Promise.reject(new Error("Could not determine where the display" +
                                             " message was from.  Widget will not be displayed"));
         } else if (cell.widget_subarea) {
             var dummy = $('<div />');
             cell.widget_subarea.append(dummy);
             return this.create_view(model, {cell: cell}).then(
                 function(view) {
                     that._handle_display_view(view);
                     dummy.replaceWith(view.$el);
                     view.trigger('displayed');
                     return view;
                 }).catch(utils.reject('Could not display view', true));
         }
     };
 
     WidgetManager.prototype._handle_display_view = function (view) {
         // Have the IPython keyboard manager disable its event
         // handling so the widget can capture keyboard input.
         // Note, this is only done on the outer most widgets.
         if (this.keyboard_manager) {
             this.keyboard_manager.register_events(view.$el);
         
             if (view.additional_elements) {
                 for (var i = 0; i < view.additional_elements.length; i++) {
                     this.keyboard_manager.register_events(view.additional_elements[i]);
                 }
             } 
         }
     };
     
     WidgetManager.prototype.create_view = function(model, options) {
         // Creates a promise for a view of a given model
 
         // Make sure the view creation is not out of order with 
         // any state updates.
         model.state_change = model.state_change.then(function() {
             
             return utils.load_class(model.get('_view_name'), model.get('_view_module'),
             WidgetManager._view_types).then(function(ViewType) {
 
                 // If a view is passed into the method, use that view's cell as
                 // the cell for the view that is created.
                 options = options || {};
                 if (options.parent !== undefined) {
                     options.cell = options.parent.options.cell;
                 }
                 // Create and render the view...
                 var parameters = {model: model, options: options};
                 var view = new ViewType(parameters);
                 view.listenTo(model, 'destroy', view.remove);
-                view.render();
-                return view;
+                return Promise.resolve(view.render()).then(function() {return view;});
             }).catch(utils.reject("Couldn't create a view for model id '" + String(model.id) + "'", true));
         });
         return model.state_change;
     };
 
     WidgetManager.prototype.get_msg_cell = function (msg_id) {
         var cell = null;
         // First, check to see if the msg was triggered by cell execution.
         if (this.notebook) {
             cell = this.notebook.get_msg_cell(msg_id);
         }
         if (cell !== null) {
             return cell;
         }
         // Second, check to see if a get_cell callback was defined
         // for the message.  get_cell callbacks are registered for
         // widget messages, so this block is actually checking to see if the
         // message was triggered by a widget.
         var kernel = this.comm_manager.kernel;
         if (kernel) {
             var callbacks = kernel.get_callbacks_for_msg(msg_id);
             if (callbacks && callbacks.iopub &&
                 callbacks.iopub.get_cell !== undefined) {
                 return callbacks.iopub.get_cell();
             }
         }
         
         // Not triggered by a cell or widget (no get_cell callback 
         // exists).
         return null;
     };
 
     WidgetManager.prototype.callbacks = function (view) {
         // callback handlers specific a view
         var callbacks = {};
         if (view && view.options.cell) {
 
             // Try to get output handlers
             var cell = view.options.cell;
             var handle_output = null;
             var handle_clear_output = null;
             if (cell.output_area) {
                 handle_output = $.proxy(cell.output_area.handle_output, cell.output_area);
                 handle_clear_output = $.proxy(cell.output_area.handle_clear_output, cell.output_area);
             }
 
             // Create callback dictionary using what is known
             var that = this;
             callbacks = {
                 iopub : {
                     output : handle_output,
                     clear_output : handle_clear_output,
 
                     // Special function only registered by widget messages.
                     // Allows us to get the cell for a message so we know
                     // where to add widgets if the code requires it.
                     get_cell : function () {
                         return cell;
                     },
                 },
             };
         }
         return callbacks;
     };
 
     WidgetManager.prototype.get_model = function (model_id) {
         // Get a promise for a model by model id.
         return this._models[model_id];
     };
 
     WidgetManager.prototype._handle_comm_open = function (comm, msg) {
         // Handle when a comm is opened.
         return this.create_model({
             model_name: msg.content.data.model_name, 
             model_module: msg.content.data.model_module, 
             comm: comm}).catch(utils.reject("Couldn't create a model.", true));
     };
 
     WidgetManager.prototype.create_model = function (options) {
         // Create and return a promise for a new widget model
         //
         // Minimally, one must provide the model_name and widget_class
         // parameters to create a model from Javascript.
         //
         // Example
         // --------
         // JS:
         // IPython.notebook.kernel.widget_manager.create_model({
         //      model_name: 'WidgetModel', 
         //      widget_class: 'IPython.html.widgets.widget_int.IntSlider'})
         //      .then(function(model) { console.log('Create success!', model); },
         //      $.proxy(console.error, console));
         //
         // Parameters
         // ----------
         // options: dictionary
         //  Dictionary of options with the following contents:
         //      model_name: string
         //          Target name of the widget model to create.
         //      model_module: (optional) string
         //          Module name of the widget model to create.
         //      widget_class: (optional) string
         //          Target name of the widget in the back-end.
         //      comm: (optional) Comm
         
         // Create a comm if it wasn't provided.
         var comm = options.comm;
         if (!comm) {
             comm = this.comm_manager.new_comm('ipython.widget', {'widget_class': options.widget_class});
         }
 
         var that = this;
         var model_id = comm.comm_id;
         var model_promise =  utils.load_class(options.model_name, options.model_module, WidgetManager._model_types)
             .then(function(ModelType) {
                 var widget_model = new ModelType(that, model_id, comm);
                 widget_model.once('comm:close', function () {
                     delete that._models[model_id];
                 });
                 return widget_model;
 
             }, function(error) {
                 delete that._models[model_id];
                 var wrapped_error = new utils.WrappedError("Couldn't create model", error);
                 return Promise.reject(wrapped_error);
             });
         this._models[model_id] = model_promise;
         return model_promise;
     };
 
     // Backwards compatibility.
     IPython.WidgetManager = WidgetManager;
 
     return {'WidgetManager': WidgetManager};
 });

docs/source/whatsnew/development.rst

@@ -1,195 +1,221 b''
 =====================
  Development version
 =====================
 
 This document describes in-flight development work.
 
 .. warning::
 
     Please do not edit this file by hand (doing so will likely cause merge
     conflicts for other Pull Requests). Instead, create a new file in the
     `docs/source/whatsnew/pr` folder
 
 Using different kernels
 -----------------------
 
 .. image:: ../_images/kernel_selector_screenshot.png
    :alt: Screenshot of notebook kernel selection dropdown menu
    :align: center
 
 You can now choose a kernel for a notebook within the user interface, rather
 than starting up a separate notebook server for each kernel you want to use. The
 syntax highlighting adapts to match the language you're working in.
 
 Information about the kernel is stored in the notebook file, so when you open a
 notebook, it will automatically start the correct kernel.
 
 It is also easier to use the Qt console and the terminal console with other
 kernels, using the --kernel flag::
 
     ipython qtconsole --kernel bash
     ipython console --kernel bash
 
     # To list available kernels
     ipython kernelspec list
 
 Kernel authors should see :ref:`kernelspecs` for how to register their kernels
 with IPython so that these mechanisms work.
 
 Typing unicode identifiers
 --------------------------
 
 .. image:: /_images/unicode_completion.png
 
 Complex expressions can be much cleaner when written with a wider choice of
 characters. Python 3 allows unicode identifiers, and IPython 3 makes it easier
 to type those, using a feature from Julia. Type a backslash followed by a LaTeX
 style short name, such as ``\alpha``. Press tab, and it will turn into α.
 
 Other new features
 ------------------
 
 * :class:`~.TextWidget` and :class:`~.TextareaWidget` objects now include a
   ``placeholder`` attribute, for displaying placeholder text before the
   user has typed anything.
 
 * The :magic:`load` magic can now find the source for objects in the user namespace.
   To enable searching the namespace, use the ``-n`` option.
 
   .. sourcecode:: ipython
 
       In [1]: %load -n my_module.some_function
 
 * :class:`~.DirectView` objects have a new :meth:`~.DirectView.use_cloudpickle`
   method, which works like ``view.use_dill()``, but causes the ``cloudpickle``
   module from PiCloud's `cloud`__ library to be used rather than dill or the
   builtin pickle module.
 
   __ https://pypi.python.org/pypi/cloud
 
 * Added a .ipynb exporter to nbconvert.  It can be used by passing `--to notebook`
   as a commandline argument to nbconvert.
 
 * New nbconvert preprocessor called :class:`~.ClearOutputPreprocessor`. This
   clears the output from IPython notebooks.
 
 * New preprocessor for nbconvert that executes all the code cells in a notebook.
   To run a notebook and save its output in a new notebook::
 
       ipython nbconvert InputNotebook --ExecutePreprocessor.enabled=True --to notebook --output Executed
 
 * Consecutive stream (stdout/stderr) output is merged into a single output
   in the notebook document.
   Previously, all output messages were preserved as separate output fields in the JSON.
   Now, the same merge is applied to the stored output as the displayed output,
   improving document load time for notebooks with many small outputs.
 
 * ``NotebookApp.webapp_settings`` is deprecated and replaced with
   the more informatively named ``NotebookApp.tornado_settings``.
 
 * Using :magic:`timeit` prints warnings if there is atleast a 4x difference in timings
   between the slowest and fastest runs, since this might meant that the multiple
   runs are not independent of one another.
 
 * It's now possible to provide mechanisms to integrate IPython with other event
   loops, in addition to the ones we already support. This lets you run GUI code
   in IPython with an interactive prompt, and to embed the IPython
   kernel in GUI applications. See :doc:`/config/eventloops` for details. As part
   of this, the direct ``enable_*`` and ``disable_*`` functions for various GUIs
   in :mod:`IPython.lib.inputhook` have been deprecated in favour of
   :meth:`~.InputHookManager.enable_gui` and :meth:`~.InputHookManager.disable_gui`.
 
 * A ``ScrollManager`` was added to the notebook.  The ``ScrollManager`` controls how the notebook document is scrolled using keyboard.  Users can inherit from the ``ScrollManager`` or ``TargetScrollManager`` to customize how their notebook scrolls.  The default ``ScrollManager`` is the ``SlideScrollManager``, which tries to scroll to the nearest slide or sub-slide cell.
 
 * The function :func:`~IPython.html.widgets.interaction.interact_manual` has been
   added which behaves similarly to :func:`~IPython.html.widgets.interaction.interact`,
   but adds a button to explicitly run the interacted-with function, rather than
   doing it automatically for every change of the parameter widgets. This should
   be useful for long-running functions.
 
 * The ``%cython`` magic is now part of the Cython module. Use `%load_ext Cython` with a version of Cython >= 0.21 to have access to the magic now.
 
 * The Notebook application now offers integrated terminals on Unix platforms,
   intended for when it is used on a remote server. To enable these, install
   the ``terminado`` Python package.
 
 * Setting the default highlighting language for nbconvert with the config option
   ``NbConvertBase.default_language`` is deprecated. Nbconvert now respects
   metadata stored in the :ref:`kernel spec <kernelspecs>`.
 
 * IPython can now be configured systemwide, with files in :file:`/etc/ipython`
   or :file:`/usr/local/etc/ipython` on Unix systems,
   or :file:`{%PROGRAMDATA%}\\ipython` on Windows.
 
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT.
 
 
 Backwards incompatible changes
 ------------------------------
 
 * :func:`IPython.core.oinspect.getsource` call specification has changed:
 
   * `oname` keyword argument has been added for property source formatting
   * `is_binary` keyword argument has been dropped, passing ``True`` had
     previously short-circuited the function to return ``None`` unconditionally
 
 * Removed the octavemagic extension: it is now available as ``oct2py.ipython``.
 
 * Creating PDFs with LaTeX no longer uses a post processor.
   Use `nbconvert --to pdf` instead of `nbconvert --to latex --post pdf`.
 
 * Used https://github.com/jdfreder/bootstrap2to3 to migrate the Notebook to Bootstrap 3.
 
   Additional changes:
 
   - Set `.tab-content .row` `0px;` left and right margin (bootstrap default is `-15px;`)
   - Removed `height: @btn_mini_height;` from `.list_header>div, .list_item>div` in `tree.less`
   - Set `#header` div `margin-bottom: 0px;`
   - Set `#menus` to `float: left;`
   - Set `#maintoolbar .navbar-text` to `float: none;`
   - Added no-padding convienence class.
   - Set border of #maintoolbar to 0px
 
 * Accessing the `container` DOM object when displaying javascript has been
   deprecated in IPython 2.0 in favor of accessing `element`. Starting with
   IPython 3.0 trying to access `container` will raise an error in browser
   javascript console.
 
 * ``IPython.utils.py3compat.open`` was removed: :func:`io.open` provides all
   the same functionality.
 
 * The NotebookManager and ``/api/notebooks`` service has been replaced by
   a more generic ContentsManager and ``/api/contents`` service,
   which supports all kinds of files.
 * The Dashboard now lists all files, not just notebooks and directories.
 * The ``--script`` hook for saving notebooks to Python scripts is removed,
   use :samp:`ipython nbconvert --to python {notebook}` instead.
 
 * The ``rmagic`` extension is deprecated, as it is now part of rpy2. See
   :mod:`rpy2.ipython.rmagic`.
 
 * :meth:`~.KernelManager.start_kernel` and :meth:`~.KernelManager.format_kernel_cmd`
   no longer accept a ``executable`` parameter. Use the kernelspec machinery instead.
 
 * The widget classes have been renamed from `*Widget` to `*`.  The old names are
   still functional, but are deprecated.  i.e. `IntSliderWidget` has been renamed
   to `IntSlider`.
 * The ContainerWidget was renamed to Box and no longer defaults as a flexible
   box in the web browser.  A new FlexBox widget was added, which allows you to
   use the flexible box model.
 
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT.
 
-IFrame embedding
-````````````````
+Content Security Policy
+```````````````````````
 
-The IPython Notebook and its APIs by default will only be allowed to be
-embedded in an iframe on the same origin.
+The Content Security Policy is a web standard for adding a layer of security to
+detect and mitigate certain classes of attacks, including Cross Site Scripting
+(XSS) and data injection attacks. This was introduced into the notebook to
+ensure that the IPython Notebook and its APIs (by default) can only be embedded
+in an iframe on the same origin.
 
-To override this, set ``headers[X-Frame-Options]`` to one of
+Override ``headers['Content-Security-Policy']`` within your notebook
+configuration to extend for alternate domains and security settings.::
 
-* DENY
-* SAMEORIGIN
-* ALLOW-FROM uri
+    c.NotebookApp.tornado_settings = {
+        'headers': {
+            'Content-Security-Policy': "frame-ancestors 'self'"
+        }
+    }
 
-See `Mozilla's guide to X-Frame-Options <https://developer.mozilla.org/en-US/docs/Web/HTTP/X-Frame-Options>`_ for more examples.
+Example policies::
+
+    Content-Security-Policy: default-src 'self' https://*.jupyter.org
+
+Matches embeddings on any subdomain of jupyter.org, so long as they are served
+over SSL.
+
+There is a `report-uri <https://developer.mozilla.org/en-US/docs/Web/Security/CSP/CSP_policy_directives#report-uri>`_ endpoint available for logging CSP violations, located at
+``/api/security/csp-report``. To use it, set ``report-uri`` as part of the CSP::
+
+    c.NotebookApp.tornado_settings = {
+        'headers': {
+            'Content-Security-Policy': "frame-ancestors 'self'; report-uri /api/security/csp-report"
+        }
+    }
+
+It simply provides the CSP report as a warning in IPython's logs. The default
+CSP sets this report-uri relative to the ``base_url`` (not shown above).
+
+For a more thorough and accurate guide on Content Security Policies, check out
+`MDN's Using Content Security Policy <https://developer.mozilla.org/en-US/docs/Web/Security/CSP/Using_Content_Security_Policy>`_ for more examples.