Merge branch 'master' of https://github.com/ipython/ipython into cython-magic-name-argument

IPython/core/tests/test_alias.py

@@ -0,0 +1,41 b''
+from IPython.utils.capture import capture_output
+
+import nose.tools as nt
+
+def test_alias_lifecycle():
+    name = 'test_alias1'
+    cmd = 'echo "Hello"'
+    am = _ip.alias_manager
+    am.clear_aliases()
+    am.define_alias(name, cmd)
+    assert am.is_alias(name)
+    nt.assert_equal(am.retrieve_alias(name), cmd)
+    nt.assert_in((name, cmd), am.aliases)
+    
+    # Test running the alias
+    orig_system = _ip.system
+    result = []
+    _ip.system = result.append
+    try:
+        _ip.run_cell('%{}'.format(name))
+        result = [c.strip() for c in result]
+        nt.assert_equal(result, [cmd])
+    finally:
+        _ip.system = orig_system
+    
+    # Test removing the alias
+    am.undefine_alias(name)
+    assert not am.is_alias(name)
+    with nt.assert_raises(ValueError):
+        am.retrieve_alias(name)
+    nt.assert_not_in((name, cmd), am.aliases)
+    
+
+def test_alias_args_error():
+    """Error expanding with wrong number of arguments"""
+    _ip.alias_manager.define_alias('parts', 'echo first %s second %s')
+    # capture stderr:
+    with capture_output() as cap:
+        _ip.run_cell('parts 1')
+
+    nt.assert_equal(cap.stderr.split(':')[0], 'UsageError')
  No newline at end of file

IPython/html/tests/test_notebookapp.py

@@ -0,0 +1,25 b''
+"""Test NotebookApp"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2013 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import nose.tools as nt
+
+import IPython.testing.tools as tt
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+
+def test_help_output():
+    """ipython notebook --help-all works"""
+    tt.help_all_output_test('notebook')
+

IPython/kernel/tests/utils.py

@@ -0,0 +1,167 b''
+"""utilities for testing IPython kernels"""
+
+#-------------------------------------------------------------------------------
+#  Copyright (C) 2013  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-------------------------------------------------------------------------------
+
+#-------------------------------------------------------------------------------
+# Imports
+#-------------------------------------------------------------------------------
+
+import atexit
+
+from contextlib import contextmanager
+from subprocess import PIPE
+from Queue import Empty
+
+import nose.tools as nt
+
+from IPython.kernel import KernelManager
+
+#-------------------------------------------------------------------------------
+# Globals
+#-------------------------------------------------------------------------------
+
+STARTUP_TIMEOUT = 60
+TIMEOUT = 15
+
+KM = None
+KC = None
+
+#-------------------------------------------------------------------------------
+# code
+#-------------------------------------------------------------------------------
+
+
+def start_new_kernel():
+    """start a new kernel, and return its Manager and Client"""
+    km = KernelManager()
+    km.start_kernel(stdout=PIPE, stderr=PIPE)
+    kc = km.client()
+    kc.start_channels()
+    
+    msg_id = kc.kernel_info()
+    kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
+    flush_channels(kc)
+    return km, kc
+
+def flush_channels(kc=None):
+    """flush any messages waiting on the queue"""
+    from .test_message_spec import validate_message
+    
+    if kc is None:
+        kc = KC
+    for channel in (kc.shell_channel, kc.iopub_channel):
+        while True:
+            try:
+                msg = channel.get_msg(block=True, timeout=0.1)
+            except Empty:
+                break
+            else:
+                validate_message(msg)
+
+
+def execute(code='', kc=None, **kwargs):
+    """wrapper for doing common steps for validating an execution request"""
+    from .test_message_spec import validate_message
+    if kc is None:
+        kc = KC
+    msg_id = kc.execute(code=code, **kwargs)
+    reply = kc.get_shell_msg(timeout=TIMEOUT)
+    validate_message(reply, 'execute_reply', msg_id)
+    busy = kc.get_iopub_msg(timeout=TIMEOUT)
+    validate_message(busy, 'status', msg_id)
+    nt.assert_equal(busy['content']['execution_state'], 'busy')
+    
+    if not kwargs.get('silent'):
+        pyin = kc.get_iopub_msg(timeout=TIMEOUT)
+        validate_message(pyin, 'pyin', msg_id)
+        nt.assert_equal(pyin['content']['code'], code)
+    
+    return msg_id, reply['content']
+
+def start_global_kernel():
+    """start the global kernel (if it isn't running) and return its client"""
+    global KM, KC
+    if KM is None:
+        KM, KC = start_new_kernel()
+        atexit.register(stop_global_kernel)
+    return KC
+
+@contextmanager
+def kernel():
+    """Context manager for the global kernel instance
+    
+    Should be used for most kernel tests
+    
+    Returns
+    -------
+    kernel_client: connected KernelClient instance
+    """
+    yield start_global_kernel()
+
+def uses_kernel(test_f):
+    """Decorator for tests that use the global kernel"""
+    def wrapped_test():
+        with kernel() as kc:
+            test_f(kc)
+    wrapped_test.__doc__ = test_f.__doc__
+    wrapped_test.__name__ = test_f.__name__
+    return wrapped_test
+
+def stop_global_kernel():
+    """Stop the global shared kernel instance, if it exists"""
+    global KM, KC
+    KC.stop_channels()
+    KC = None
+    if KM is None:
+        return
+    KM.shutdown_kernel(now=True)
+    KM = None
+
+@contextmanager
+def new_kernel():
+    """Context manager for a new kernel in a subprocess
+    
+    Should only be used for tests where the kernel must not be re-used.
+    
+    Returns
+    -------
+    kernel_client: connected KernelClient instance
+    """
+    km, kc = start_new_kernel()
+    try:
+        yield kc
+    finally:
+        kc.stop_channels()
+        km.shutdown_kernel(now=True)
+
+
+def assemble_output(iopub):
+    """assemble stdout/err from an execution"""
+    stdout = ''
+    stderr = ''
+    while True:
+        msg = iopub.get_msg(block=True, timeout=1)
+        msg_type = msg['msg_type']
+        content = msg['content']
+        if msg_type == 'status' and content['execution_state'] == 'idle':
+            # idle message signals end of output
+            break
+        elif msg['msg_type'] == 'stream':
+            if content['name'] == 'stdout':
+                stdout += content['data']
+            elif content['name'] == 'stderr':
+                stderr += content['data']
+            else:
+                raise KeyError("bad stream: %r" % content['name'])
+        else:
+            # other output, ignored
+            pass
+    return stdout, stderr
+
+
+

IPython/nbconvert/exporters/templateexporter.py

@@ -0,0 +1,316 b''
+"""This module defines Exporter, a highly configurable converter
+that uses Jinja2 to export notebook files into different formats.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function, absolute_import
+
+# Stdlib imports
+import os
+
+# other libs/dependencies
+from jinja2 import Environment, FileSystemLoader, ChoiceLoader, TemplateNotFound
+
+# IPython imports
+from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Dict, Any
+from IPython.utils.importstring import import_item
+from IPython.utils import py3compat, text
+
+from IPython.nbconvert import filters
+from .exporter import Exporter
+
+#-----------------------------------------------------------------------------
+# Globals and constants
+#-----------------------------------------------------------------------------
+
+#Jinja2 extensions to load.
+JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']
+
+default_filters = {
+        'indent': text.indent,
+        'markdown2html': filters.markdown2html,
+        'ansi2html': filters.ansi2html,
+        'filter_data_type': filters.DataTypeFilter,
+        'get_lines': filters.get_lines,
+        'highlight2html': filters.highlight2html,
+        'highlight2latex': filters.highlight2latex,
+        'ipython2python': filters.ipython2python,
+        'posix_path': filters.posix_path,
+        'markdown2latex': filters.markdown2latex,
+        'markdown2rst': filters.markdown2rst,
+        'comment_lines': filters.comment_lines,
+        'strip_ansi': filters.strip_ansi,
+        'strip_dollars': filters.strip_dollars,
+        'strip_files_prefix': filters.strip_files_prefix,
+        'html2text' : filters.html2text,
+        'add_anchor': filters.add_anchor,
+        'ansi2latex': filters.ansi2latex,
+        'strip_math_space': filters.strip_math_space,
+        'wrap_text': filters.wrap_text,
+        'escape_latex': filters.escape_latex,
+        'citation2latex': filters.citation2latex,
+        'path2url': filters.path2url,
+        'add_prompts': filters.add_prompts,
+}
+
+#-----------------------------------------------------------------------------
+# Class
+#-----------------------------------------------------------------------------
+
+class TemplateExporter(Exporter):
+    """
+    Exports notebooks into other file formats.  Uses Jinja 2 templating engine
+    to output new formats.  Inherit from this class if you are creating a new
+    template type along with new filters/preprocessors.  If the filters/
+    preprocessors provided by default suffice, there is no need to inherit from
+    this class.  Instead, override the template_file and file_extension
+    traits via a config file.
+
+    {filters}
+    """
+    
+    # finish the docstring
+    __doc__ = __doc__.format(filters = '- '+'\n    - '.join(default_filters.keys()))
+
+
+    template_file = Unicode(u'default',
+            config=True,
+            help="Name of the template file to use")
+    def _template_file_changed(self, name, old, new):
+        if new == 'default':
+            self.template_file = self.default_template
+        else:
+            self.template_file = new
+        self.template = None
+        self._load_template()
+    
+    default_template = Unicode(u'')
+    template = Any()
+    environment = Any()
+
+    template_path = List(['.'], config=True)
+    def _template_path_changed(self, name, old, new):
+        self._load_template()
+
+    default_template_path = Unicode(
+        os.path.join("..", "templates"), 
+        help="Path where the template files are located.")
+
+    template_skeleton_path = Unicode(
+        os.path.join("..", "templates", "skeleton"), 
+        help="Path where the template skeleton files are located.") 
+
+    #Jinja block definitions
+    jinja_comment_block_start = Unicode("", config=True)
+    jinja_comment_block_end = Unicode("", config=True)
+    jinja_variable_block_start = Unicode("", config=True)
+    jinja_variable_block_end = Unicode("", config=True)
+    jinja_logic_block_start = Unicode("", config=True)
+    jinja_logic_block_end = Unicode("", config=True)
+    
+    #Extension that the template files use.    
+    template_extension = Unicode(".tpl", config=True)
+
+    filters = Dict(config=True,
+        help="""Dictionary of filters, by name and namespace, to add to the Jinja
+        environment.""")
+
+
+    def __init__(self, config=None, extra_loaders=None, **kw):
+        """
+        Public constructor
+    
+        Parameters
+        ----------
+        config : config
+            User configuration instance.
+        extra_loaders : list[of Jinja Loaders]
+            ordered list of Jinja loader to find templates. Will be tried in order
+            before the default FileSystem ones.
+        template : str (optional, kw arg)
+            Template to use when exporting.
+        """
+        if not config:
+            config = self.default_config
+        
+        super(Exporter, self).__init__(config=config, **kw)
+
+        #Init
+        self._init_template()
+        self._init_environment(extra_loaders=extra_loaders)
+        self._init_preprocessors()
+        self._init_filters()
+
+
+    def _load_template(self):
+        """Load the Jinja template object from the template file
+        
+        This is a no-op if the template attribute is already defined,
+        or the Jinja environment is not setup yet.
+        
+        This is triggered by various trait changes that would change the template.
+        """
+        if self.template is not None:
+            return
+        # called too early, do nothing
+        if self.environment is None:
+            return
+        # Try different template names during conversion.  First try to load the
+        # template by name with extension added, then try loading the template
+        # as if the name is explicitly specified, then try the name as a 
+        # 'flavor', and lastly just try to load the template by module name.
+        module_name = self.__module__.rsplit('.', 1)[-1]
+        try_names = []
+        if self.template_file:
+            try_names.extend([
+                self.template_file + self.template_extension,
+                self.template_file,
+                module_name + '_' + self.template_file + self.template_extension,
+            ])
+        try_names.append(module_name + self.template_extension)
+        for try_name in try_names:
+            self.log.debug("Attempting to load template %s", try_name)
+            try:
+                self.template = self.environment.get_template(try_name)
+            except (TemplateNotFound, IOError):
+                pass
+            except Exception as e:
+                self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True)
+            else:
+                self.log.info("Loaded template %s", try_name)
+                break
+    
+    def from_notebook_node(self, nb, resources=None, **kw):
+        """
+        Convert a notebook from a notebook node instance.
+    
+        Parameters
+        ----------
+        nb : Notebook node
+        resources : dict (**kw) 
+            of additional resources that can be accessed read/write by 
+            preprocessors and filters.
+        """
+        nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
+
+        self._load_template()
+
+        if self.template is not None:
+            output = self.template.render(nb=nb_copy, resources=resources)
+        else:
+            raise IOError('template file "%s" could not be found' % self.template_file)
+        return output, resources
+
+
+    def register_filter(self, name, jinja_filter):
+        """
+        Register a filter.
+        A filter is a function that accepts and acts on one string.  
+        The filters are accesible within the Jinja templating engine.
+    
+        Parameters
+        ----------
+        name : str
+            name to give the filter in the Jinja engine
+        filter : filter
+        """
+        if jinja_filter is None:
+            raise TypeError('filter')
+        isclass = isinstance(jinja_filter, type)
+        constructed = not isclass
+
+        #Handle filter's registration based on it's type
+        if constructed and isinstance(jinja_filter, py3compat.string_types):
+            #filter is a string, import the namespace and recursively call
+            #this register_filter method
+            filter_cls = import_item(jinja_filter)
+            return self.register_filter(name, filter_cls)
+        
+        if constructed and hasattr(jinja_filter, '__call__'):
+            #filter is a function, no need to construct it.
+            self.environment.filters[name] = jinja_filter
+            return jinja_filter
+
+        elif isclass and isinstance(jinja_filter, MetaHasTraits):
+            #filter is configurable.  Make sure to pass in new default for 
+            #the enabled flag if one was specified.
+            filter_instance = jinja_filter(parent=self)
+            self.register_filter(name, filter_instance )
+
+        elif isclass:
+            #filter is not configurable, construct it
+            filter_instance = jinja_filter()
+            self.register_filter(name, filter_instance)
+
+        else:
+            #filter is an instance of something without a __call__ 
+            #attribute.  
+            raise TypeError('filter')
+
+        
+    def _init_template(self):
+        """
+        Make sure a template name is specified.  If one isn't specified, try to
+        build one from the information we know.
+        """
+        self._template_file_changed('template_file', self.template_file, self.template_file)
+        
+
+    def _init_environment(self, extra_loaders=None):
+        """
+        Create the Jinja templating environment.
+        """
+        here = os.path.dirname(os.path.realpath(__file__))
+        loaders = []
+        if extra_loaders:
+            loaders.extend(extra_loaders)
+
+        paths = self.template_path
+        paths.extend([os.path.join(here, self.default_template_path),
+                      os.path.join(here, self.template_skeleton_path)])
+        loaders.append(FileSystemLoader(paths))
+
+        self.environment = Environment(
+            loader= ChoiceLoader(loaders),
+            extensions=JINJA_EXTENSIONS
+            )
+        
+        #Set special Jinja2 syntax that will not conflict with latex.
+        if self.jinja_logic_block_start:
+            self.environment.block_start_string = self.jinja_logic_block_start
+        if self.jinja_logic_block_end:
+            self.environment.block_end_string = self.jinja_logic_block_end
+        if self.jinja_variable_block_start:
+            self.environment.variable_start_string = self.jinja_variable_block_start
+        if self.jinja_variable_block_end:
+            self.environment.variable_end_string = self.jinja_variable_block_end
+        if self.jinja_comment_block_start:
+            self.environment.comment_start_string = self.jinja_comment_block_start
+        if self.jinja_comment_block_end:
+            self.environment.comment_end_string = self.jinja_comment_block_end
+
+    
+    def _init_filters(self):
+        """
+        Register all of the filters required for the exporter.
+        """
+        
+        #Add default filters to the Jinja2 environment
+        for key, value in default_filters.items():
+            self.register_filter(key, value)
+
+        #Load user filters.  Overwrite existing filters if need be.
+        if self.filters:
+            for key, user_filter in self.filters.items():
+                self.register_filter(key, user_filter)

IPython/nbconvert/exporters/tests/test_templateexporter.py

@@ -0,0 +1,108 b''
+"""
+Module with tests for templateexporter.py
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from IPython.config import Config
+
+from .base import ExportersTestsBase
+from .cheese import CheesePreprocessor
+from ..templateexporter import TemplateExporter
+
+
+#-----------------------------------------------------------------------------
+# Class
+#-----------------------------------------------------------------------------
+
+class TestExporter(ExportersTestsBase):
+    """Contains test functions for exporter.py"""
+
+
+    def test_constructor(self):
+        """
+        Can a TemplateExporter be constructed?
+        """
+        TemplateExporter()
+
+
+    def test_export(self):
+        """
+        Can a TemplateExporter export something?
+        """
+        exporter = self._make_exporter()
+        (output, resources) = exporter.from_filename(self._get_notebook())
+        assert len(output) > 0
+
+
+    def test_extract_outputs(self):
+        """
+        If the ExtractOutputPreprocessor is enabled, are outputs extracted?
+        """
+        config = Config({'ExtractOutputPreprocessor': {'enabled': True}})
+        exporter = self._make_exporter(config=config)
+        (output, resources) = exporter.from_filename(self._get_notebook())
+        assert resources is not None
+        assert isinstance(resources['outputs'], dict)
+        assert len(resources['outputs']) > 0
+
+
+    def test_preprocessor_class(self):
+        """
+        Can a preprocessor be added to the preprocessors list by class type?
+        """
+        config = Config({'Exporter': {'preprocessors': [CheesePreprocessor]}})
+        exporter = self._make_exporter(config=config)
+        (output, resources) = exporter.from_filename(self._get_notebook())
+        assert resources is not None
+        assert resources['cheese'] == 'real'
+
+
+    def test_preprocessor_instance(self):
+        """
+        Can a preprocessor be added to the preprocessors list by instance?
+        """
+        config = Config({'Exporter': {'preprocessors': [CheesePreprocessor()]}})
+        exporter = self._make_exporter(config=config)
+        (output, resources) = exporter.from_filename(self._get_notebook())
+        assert resources is not None
+        assert resources['cheese'] == 'real'
+
+
+    def test_preprocessor_dottedobjectname(self):
+        """
+        Can a preprocessor be added to the preprocessors list by dotted object name?
+        """
+        config = Config({'Exporter': {'preprocessors': ['IPython.nbconvert.exporters.tests.cheese.CheesePreprocessor']}})
+        exporter = self._make_exporter(config=config)
+        (output, resources) = exporter.from_filename(self._get_notebook())
+        assert resources is not None
+        assert resources['cheese'] == 'real'
+
+
+    def test_preprocessor_via_method(self):
+        """
+        Can a preprocessor be added via the Exporter convenience method?
+        """
+        exporter = self._make_exporter()
+        exporter.register_preprocessor(CheesePreprocessor, enabled=True)
+        (output, resources) = exporter.from_filename(self._get_notebook())
+        assert resources is not None
+        assert resources['cheese'] == 'real'
+
+
+    def _make_exporter(self, config=None):
+        # Create the exporter instance, make sure to set a template name since
+        # the base TemplateExporter doesn't have a template associated with it.
+        exporter = TemplateExporter(config=config, template_file='python')
+        return exporter        

IPython/nbconvert/filters/citation.py

@@ -0,0 +1,72 b''
+"""Citation handling for LaTeX output."""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+__all__ = ['citation2latex']
+
+
+def citation2latex(s):
+    """Parse citations in Markdown cells.
+    
+    This looks for HTML tags having a data attribute names `data-cite`
+    and replaces it by the call to LaTeX cite command. The tranformation
+    looks like this:
+    
+    `<cite data-cite="granger">(Granger, 2013)</cite>`
+    
+    Becomes
+    
+    `\\cite{granger}`
+    
+    Any HTML tag can be used, which allows the citations to be formatted
+    in HTML in any manner.
+    """
+    try:
+        from lxml import html
+    except ImportError:
+        return s
+
+    tree = html.fragment_fromstring(s, create_parent='div')
+    _process_node_cite(tree)
+    s = html.tostring(tree, encoding='unicode')
+    if s.endswith('</div>'):
+        s = s[:-6]
+    if s.startswith('<div>'):
+        s = s[5:]
+    return s
+
+
+def _process_node_cite(node):
+    """Do the citation replacement as we walk the lxml tree."""
+    
+    def _get(o, name):
+        value = getattr(o, name, None)
+        return '' if value is None else value
+    
+    if 'data-cite' in node.attrib:
+        cite = '\cite{%(ref)s}' % {'ref': node.attrib['data-cite']}
+        prev = node.getprevious()
+        if prev is not None:
+            prev.tail = _get(prev, 'tail') + cite + _get(node, 'tail')
+        else:
+            parent = node.getparent()
+            if parent is not None:
+                parent.text = _get(parent, 'text') + cite + _get(node, 'tail')
+        try:
+            node.getparent().remove(node)
+        except AttributeError:
+            pass
+    else:
+        for child in node:
+            _process_node_cite(child)

IPython/nbconvert/filters/tests/test_citation.py

@@ -0,0 +1,58 b''
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from ..citation import citation2latex
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+test_md = """
+# My Heading
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus ac magna non augue
+porttitor scelerisque ac id diam <cite data-cite="granger">Granger</cite>. Mauris elit
+velit, lobortis sed interdum at, vestibulum vitae libero <strong data-cite="fperez">Perez</strong>.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit
+<em data-cite="takluyver">Thomas</em>. Quisque iaculis ligula ut ipsum mattis viverra.
+
+<p>Here is a plain paragraph that should be unaffected.</p>
+
+* One <cite data-cite="jdfreder">Jonathan</cite>.
+* Two <cite data-cite="carreau">Matthias</cite>.
+* Three <cite data-cite="ivanov">Paul</cite>.
+"""
+
+test_md_parsed = """
+# My Heading
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus ac magna non augue
+porttitor scelerisque ac id diam \cite{granger}. Mauris elit
+velit, lobortis sed interdum at, vestibulum vitae libero \cite{fperez}.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit
+\cite{takluyver}. Quisque iaculis ligula ut ipsum mattis viverra.
+
+<p>Here is a plain paragraph that should be unaffected.</p>
+
+* One \cite{jdfreder}.
+* Two \cite{carreau}.
+* Three \cite{ivanov}.
+"""
+
+def test_citation2latex():
+    """Are citations parsed properly?"""
+    try:
+        import lxml
+    except ImportError:
+        assert test_md == citation2latex(test_md) 
+    else:
+        assert test_md_parsed == citation2latex(test_md)

IPython/nbconvert/preprocessors/highlightmagics.py

@@ -0,0 +1,113 b''
+"""This preprocessor detect cells using a different language through
+magic extensions such as `%%R` or `%%octave`. Cell's metadata is marked
+so that the appropriate highlighter can be used in the `highlight`
+filter.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function, absolute_import
+
+import re
+
+# Our own imports
+from .base import Preprocessor
+from IPython.utils.traitlets import Dict
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class HighlightMagicsPreprocessor(Preprocessor):
+    """
+    Detects and tags code cells that use a different languages than Python.
+    """
+
+    # list of magic language extensions and their associated pygment lexers
+    default_languages = Dict(
+        default_value={
+            '%%R': 'r',
+            '%%bash': 'bash',
+            '%%cython': 'cython',
+            '%%javascript': 'javascript',
+            '%%julia': 'julia',
+            '%%latex': 'latex',
+            '%%octave': 'octave',
+            '%%perl': 'perl',
+            '%%ruby': 'ruby',
+            '%%sh': 'sh'})
+
+    # user defined language extensions
+    languages = Dict(
+        config=True,
+        help=("Syntax highlighting for magic's extension languages. "
+         "Each item associates a language magic extension such as %%R, "
+         "with a pygments lexer such as r."))
+
+    def __init__(self, config=None, **kw):
+        """Public constructor"""
+
+        super(HighlightMagicsPreprocessor, self).__init__(config=config, **kw)
+
+        # Update the default languages dict with the user configured ones
+        self.default_languages.update(self.languages)
+
+        # build a regular expression to catch language extensions and choose
+        # an adequate pygments lexer
+        any_language = "|".join(self.default_languages.keys())
+        self.re_magic_language = re.compile(
+            r'^\s*({0})\s+'.format(any_language))
+
+    def which_magic_language(self, source):
+        """
+        When a cell uses another language through a magic extension,
+        the other language is returned.
+        If no language magic is detected, this function returns None.
+
+        Parameters
+        ----------
+        source: str
+            Source code of the cell to highlight
+        """
+
+        m = self.re_magic_language.match(source)
+
+        if m:
+            # By construction of the re, the matched language must be in the
+            # languages dictionary
+            return self.default_languages[m.group(1)]
+        else:
+            return None
+
+    def preprocess_cell(self, cell, resources, cell_index):
+        """
+        Tags cells using a magic extension language
+
+        Parameters
+        ----------
+        cell : NotebookNode cell
+            Notebook cell being processed
+        resources : dictionary
+            Additional resources used in the conversion process.  Allows
+            preprocessors to pass variables into the Jinja engine.
+        cell_index : int
+            Index of the cell being processed (see base.py)
+        """
+
+        # Only tag code cells
+        if hasattr(cell, "input") and cell.cell_type == "code":
+            magic_language = self.which_magic_language(cell.input)
+            if magic_language:
+                cell['metadata']['magics_language'] = magic_language
+        return cell, resources

IPython/nbconvert/templates/latex/latex_base.tplx

@@ -0,0 +1,197 b''
+((= Latex base template (must inherit)
+This template builds upon the abstract template, adding common latex output
+functions.  Figures, data_text,
+This template does not define a docclass, the inheriting class must define this.=))
+
+((*- extends 'display_priority.tplx' -*))
+
+%===============================================================================
+% Abstract overrides
+%===============================================================================
+
+((* block header *))
+    ((* block docclass *))((* endblock docclass *))
+    
+    ((* block packages *))
+    \usepackage{graphicx} % Used to insert images
+    \usepackage{adjustbox} % Used to constrain images to a maximum size 
+    \usepackage{color} % Allow colors to be defined
+    \usepackage{enumerate} % Needed for markdown enumerations to work
+    \usepackage{geometry} % Used to adjust the document margins
+    \usepackage{amsmath} % Equations
+    \usepackage{amssymb} % Equations
+    \usepackage[utf8]{inputenc} % Allow utf-8 characters in the tex document
+    \usepackage{ucs} % Extended unicode (utf-8) support
+    \usepackage{fancyvrb} % verbatim replacement that allows latex
+    \usepackage{grffile} % extends the file name processing of package graphics 
+                         % to support a larger range 
+    % The hyperref package gives us a pdf with properly built
+    % internal navigation ('pdf bookmarks' for the table of contents,
+    % internal cross-reference links, web links for URLs, etc.)
+    \usepackage{hyperref}
+    ((* endblock packages *))
+
+    ((* block definitions *))
+    \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}
+    \definecolor{darkorange}{rgb}{.71,0.21,0.01}
+    \definecolor{darkgreen}{rgb}{.12,.54,.11}
+    \definecolor{myteal}{rgb}{.26, .44, .56}
+    \definecolor{gray}{gray}{0.45}
+    \definecolor{lightgray}{gray}{.95}
+    \definecolor{mediumgray}{gray}{.8}
+    \definecolor{inputbackground}{rgb}{.95, .95, .85}
+    \definecolor{outputbackground}{rgb}{.95, .95, .95}
+    \definecolor{traceback}{rgb}{1, .95, .95}
+    % new ansi colors
+    \definecolor{brown}{rgb}{0.54,0.27,0.07}
+    \definecolor{purple}{rgb}{0.5,0.0,0.5}
+    \definecolor{darkgray}{gray}{0.25}
+    \definecolor{lightred}{rgb}{1.0,0.39,0.28}
+    \definecolor{lightgreen}{rgb}{0.48,0.99,0.0}
+    \definecolor{lightblue}{rgb}{0.53,0.81,0.92}
+    \definecolor{lightpurple}{rgb}{0.87,0.63,0.87}
+    \definecolor{lightcyan}{rgb}{0.5,1.0,0.83}
+    % Define a nice break command that doesn't care if a line doesn't already
+    % exist.
+    \def\br{\hspace*{\fill} \\* }
+    % Math Jax compatability definitions
+    \def\gt{>}
+    \def\lt{<}
+    % Document parameters
+    ((* block title *))\title{((( resources.metadata.name | escape_latex )))}((* endblock title *))
+    ((* block date *))((* endblock date *))
+    ((* block author *))((* endblock author *))
+    ((* endblock definitions *))
+
+    ((* block commands *))
+    % Prevent overflowing lines due to hard-to-break entities
+    \sloppy 
+    % Setup hyperref package
+    \hypersetup{
+      breaklinks=true,  % so long urls are correctly broken across lines
+      colorlinks=true,
+      urlcolor=blue,
+      linkcolor=darkorange,
+      citecolor=darkgreen,
+      }
+    % Slightly bigger margins than the latex defaults
+    ((* block margins *))
+    \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
+    ((* endblock margins *))
+    ((* endblock commands *))
+((* endblock header *))
+
+((* block body *))
+    \begin{document}
+    
+    ((* block predoc *))
+    ((* block maketitle *))\maketitle((* endblock maketitle *))
+    ((* block abstract *))((* endblock abstract *))
+    ((* endblock predoc *))
+
+    ((( super() )))
+
+    % Add a bibliography block to the postdoc
+    ((* block postdoc *))
+    ((* block bibliography *))((* endblock bibliography *))
+    ((* endblock postdoc *))
+    \end{document}
+((* endblock body *))
+
+%===============================================================================
+% Support blocks
+%===============================================================================
+
+% Displaying simple data text
+((* block data_text *))
+    \begin{verbatim}
+((( output.text )))
+    \end{verbatim}
+((* endblock data_text *))
+
+% Display python error text as-is
+((* block pyerr *))
+    \begin{Verbatim}[commandchars=\\\{\}]
+((( super() )))
+    \end{Verbatim}
+((* endblock pyerr *))
+((* block traceback_line *))
+    ((( line | indent | strip_ansi | escape_latex )))
+((* endblock traceback_line *))
+
+% Display stream ouput with coloring
+((* block stream *))
+    \begin{Verbatim}[commandchars=\\\{\}]
+((( output.text | escape_latex | ansi2latex )))
+    \end{Verbatim}
+((* endblock stream *))
+
+% Display latex
+((* block data_latex -*))
+    ((*- if output.latex.startswith('$'): -*)) 
+        ((= Replace $ symbols with more explicit, equation block. =))
+        \begin{equation*}
+        ((( output.latex | strip_dollars )))
+        \end{equation*}
+    ((*- else -*))
+        ((( output.latex )))
+    ((*- endif *))
+((* endblock data_latex *))
+
+% Default mechanism for rendering figures
+((*- block data_png -*))((( draw_figure(output.png_filename) )))((*- endblock -*))
+((*- block data_jpg -*))((( draw_figure(output.jpeg_filename) )))((*- endblock -*))
+((*- block data_svg -*))((( draw_figure(output.svg_filename) )))((*- endblock -*))
+((*- block data_pdf -*))((( draw_figure(output.pdf_filename) )))((*- endblock -*))
+
+% Draw a figure using the graphicx package.
+((* macro draw_figure(filename) -*))
+((* set filename = filename | posix_path *))
+((*- block figure scoped -*))
+    \begin{center}
+    \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{((( filename )))}
+    \end{center}
+    { \hspace*{\fill} \\}
+((*- endblock figure -*))
+((*- endmacro *))
+
+% Draw heading cell.  Explicitly map different cell levels.
+((* block headingcell scoped *))
+
+    ((* if cell.level == 1 -*))
+        ((* block h1 -*))\section((* endblock h1 -*))
+    ((* elif cell.level == 2 -*))
+        ((* block h2 -*))\subsection((* endblock h2 -*))
+    ((* elif cell.level == 3 -*))
+        ((* block h3 -*))\subsubsection((* endblock h3 -*))
+    ((* elif cell.level == 4 -*))
+        ((* block h4 -*))\paragraph((* endblock h4 -*))
+    ((* elif cell.level == 5 -*))
+        ((* block h5 -*))\subparagraph((* endblock h5 -*))
+    ((* elif cell.level == 6 -*))
+        ((* block h6 -*))\\*\textit((* endblock h6 -*))
+    ((*- endif -*))
+    {((( cell.source | replace('\n', ' ') | citation2latex | markdown2latex )))}
+
+((* endblock headingcell *))
+
+% Redirect pyout to display data priority.
+((* block pyout scoped *))
+    ((* block data_priority scoped *))
+    ((( super() )))
+    ((* endblock *))
+((* endblock pyout *))
+
+% Render markdown
+((* block markdowncell scoped *))
+    ((( cell.source | citation2latex | markdown2latex )))
+((* endblock markdowncell *))
+
+% Spit out the contents of raw cells unmodified
+((* block rawcell scoped *))
+    ((( cell.source )))
+((* endblock rawcell *))
+
+% Don't display unknown types
+((* block unknowncell scoped *))
+((* endblock unknowncell *))

IPython/nbconvert/templates/latex/latex_report.tplx

@@ -0,0 +1,22 b''
+
+% Default to the notebook output style
+((* if not cell_style is defined *))
+    ((* set cell_style = 'style_ipython.tplx' *))
+((* endif *))
+
+% Inherit from the specified cell style.
+((* extends cell_style *))
+
+
+%===============================================================================
+% Latex Book
+%===============================================================================
+
+((* block predoc *))
+    ((( super() )))
+    ((* block tableofcontents *))\tableofcontents((* endblock tableofcontents *))
+((* endblock predoc *))
+
+((* block docclass *))
+\documentclass{report}
+((* endblock docclass *))

IPython/nbconvert/templates/latex/style_bw_ipython.tplx

@@ -0,0 +1,41 b''
+((= Black&white ipython input/output style =))
+
+((*- extends 'latex_base.tplx' -*))
+
+%===============================================================================
+% Input
+%===============================================================================
+
+((* block input scoped *))
+((( add_prompt(cell.input, cell, 'In ') )))
+((* endblock input *))
+
+
+%===============================================================================
+% Output
+%===============================================================================
+
+((* block pyout scoped *))
+    ((*- for type in output | filter_data_type -*))
+        ((*- if type in ['text']*))
+((( add_prompt(output.text, cell, 'Out') )))
+        ((*- else -*))
+\verb+Out[((( cell.prompt_number )))]:+((( super() )))
+        ((*- endif -*))
+    ((*- endfor -*))
+((* endblock pyout *))
+
+
+%==============================================================================
+% Support Macros
+%==============================================================================
+
+% Name: draw_prompt
+% Purpose: Renders an output/input prompt
+((* macro add_prompt(text, cell, prompt) -*))
+    ((*- set prompt_number = "" ~ cell.prompt_number -*))
+    ((*- set indentation = " " * (prompt_number | length + 7) -*))
+\begin{verbatim}
+(((- text | add_prompts(first=prompt ~ '[' ~ prompt_number ~ ']: ', cont=indentation) -)))
+\end{verbatim}
+((*- endmacro *))

IPython/nbconvert/templates/latex/style_bw_python.tplx

@@ -0,0 +1,13 b''
+((= Black&white Python input/output style =))
+
+((*- extends 'latex_base.tplx' -*))
+
+%===============================================================================
+% Input
+%===============================================================================
+
+((* block input scoped *))
+\begin{verbatim}
+((( cell.input | add_prompts )))
+\end{verbatim}
+((* endblock input *))

IPython/nbconvert/templates/latex/style_ipython.tplx

@@ -0,0 +1,54 b''
+((= IPython input/output style =))
+
+((*- extends 'latex_base.tplx' -*))
+
+% Custom definitions
+((* block definitions *))
+    ((( super() )))
+
+    % Pygments definitions
+    ((( resources.latex.pygments_definitions )))
+
+    % Exact colors from NB
+    \definecolor{incolor}{rgb}{0.0, 0.0, 0.5}
+    \definecolor{outcolor}{rgb}{0.545, 0.0, 0.0}
+
+((* endblock definitions *))
+
+%===============================================================================
+% Input
+%===============================================================================
+
+((* block input scoped *))
+    ((( add_prompt(cell.input | highlight2latex(strip_verbatim=True), cell, 'In ', 'incolor') )))
+((* endblock input *))
+
+
+%===============================================================================
+% Output
+%===============================================================================
+
+((* block pyout scoped *))
+    ((*- for type in output | filter_data_type -*))
+        ((*- if type in ['text']*))
+            ((( add_prompt(output.text | escape_latex, cell, 'Out', 'outcolor') )))
+        ((* else -*))
+\texttt{\color{outcolor}Out[{\color{outcolor}((( cell.prompt_number )))}]:}((( super() )))
+        ((*- endif -*))
+    ((*- endfor -*))
+((* endblock pyout *))
+
+
+%==============================================================================
+% Support Macros
+%==============================================================================
+
+% Name: draw_prompt
+% Purpose: Renders an output/input prompt
+((* macro add_prompt(text, cell, prompt, prompt_color) -*))
+    ((*- set prompt_number = "" ~ cell.prompt_number -*))
+    ((*- set indention =  " " * (prompt_number | length + 7) -*))
+\begin{Verbatim}[commandchars=\\\{\}]
+((( text | add_prompts(first='{\color{' ~ prompt_color ~ '}' ~ prompt ~ '[{\\color{' ~ prompt_color ~ '}' ~ prompt_number ~ '}]:} ', cont=indention) )))
+\end{Verbatim}
+((*- endmacro *))

IPython/nbconvert/templates/latex/style_python.tplx

@@ -0,0 +1,21 b''
+((= Python input/output style =))
+
+((*- extends 'latex_base.tplx' -*))
+
+% Custom definitions
+((* block definitions *))
+    ((( super() )))
+
+    % Pygments definitions
+    ((( resources.latex.pygments_definitions )))
+((* endblock definitions *))
+
+%===============================================================================
+% Input
+%===============================================================================
+
+((* block input scoped *))
+    \begin{Verbatim}[commandchars=\\\{\}]
+((( cell.input | highlight2latex(strip_verbatim=True) | add_prompts )))
+    \end{Verbatim}
+((* endblock input *))

.travis.yml

@@ -4,13 +4,11 b' python:'
     - 2.7
     - 3.3
 before_install:
-    - pip install jinja2
     - easy_install -q pyzmq
+    - pip install jinja2 sphinx pygments tornado
     - sudo apt-get install pandoc
-    - pip install pygments
-    - pip install sphinx
 install:
     - python setup.py install -q 
 script:
-    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then iptest -w /tmp; fi
-    - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then iptest3 -w /tmp; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cd /tmp; iptest; fi
+    - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then cd /tmp; iptest3; fi

IPython/__init__.py

@@ -28,8 +28,8 b' import sys'
 #-----------------------------------------------------------------------------
 
 # Don't forget to also update setup.py when this changes!
-if sys.version[0:3] < '2.6':
-    raise ImportError('Python Version 2.6 or above is required for IPython.')
+if sys.version_info[:2] < (2,7):
+    raise ImportError('IPython requires Python Version 2.7 or above.')
 
 # Make it easy to import extensions - they are always directly on pythonpath.
 # Therefore, non-IPython modules can be added to extensions directory.

IPython/config/application.py

@@ -142,6 +142,9 b' class Application(SingletonConfigurable):'
 
     # The version string of this application.
     version = Unicode(u'0.0')
+    
+    # the argv used to initialize the application
+    argv = List()
 
     # The log level for the application
     log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
@@ -454,6 +457,7 b' class Application(SingletonConfigurable):'
     def parse_command_line(self, argv=None):
         """Parse the command line arguments."""
         argv = sys.argv[1:] if argv is None else argv
+        self.argv = list(argv)
         
         if argv and argv[0] == 'help':
             # turn `ipython help notebook` into `ipython notebook -h`

IPython/config/loader.py

@@ -24,11 +24,11 b' Authors'
 #-----------------------------------------------------------------------------
 
 import __builtin__ as builtin_mod
+import argparse
 import os
 import re
 import sys
 
-from IPython.external import argparse
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils import py3compat, warn
 from IPython.utils.encoding import DEFAULT_ENCODING

IPython/config/tests/test_application.py

@@ -1,3 +1,4 b''
+# coding: utf-8
 """
 Tests for IPython.config.application.Application
 
@@ -185,4 +186,8 b' class TestApplication(TestCase):'
         self.assertEqual(app.bar.b, 5)
         self.assertEqual(app.extra_args, ['extra', '--disable', 'args'])
     
+    def test_unicode_argv(self):
+        app = MyApp()
+        app.parse_command_line(['ünîcødé'])
+        
 

IPython/consoleapp.py

@@ -53,7 +53,7 b' from IPython.kernel.connect import ConnectionFileMixin'
 # Network Constants
 #-----------------------------------------------------------------------------
 
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 
 #-----------------------------------------------------------------------------
 # Globals
@@ -254,7 +254,7 b' class IPythonConsoleApp(ConnectionFileMixin):'
         with open(fname) as f:
             cfg = json.load(f)
         self.transport = cfg.get('transport', 'tcp')
-        self.ip = cfg.get('ip', LOCALHOST)
+        self.ip = cfg.get('ip', localhost())
         
         for channel in ('hb', 'shell', 'iopub', 'stdin', 'control'):
             name = channel + '_port'
@@ -282,7 +282,7 b' class IPythonConsoleApp(ConnectionFileMixin):'
         if self.sshkey and not self.sshserver:
             # specifying just the key implies that we are connecting directly
             self.sshserver = ip
-            ip = LOCALHOST
+            ip = localhost()
         
         # build connection dict for tunnels:
         info = dict(ip=ip,
@@ -295,7 +295,7 b' class IPythonConsoleApp(ConnectionFileMixin):'
         self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver))
         
         # tunnels return a new set of ports, which will be on localhost:
-        self.ip = LOCALHOST
+        self.ip = localhost()
         try:
             newports = tunnel_to_kernel(info, self.sshserver, self.sshkey)
         except:

IPython/core/alias.py

@@ -20,17 +20,15 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
-import __builtin__
-import keyword
 import os
 import re
 import sys
 
 from IPython.config.configurable import Configurable
-from IPython.core.splitinput import split_user_input
+from IPython.core.error import UsageError
 
 from IPython.utils.traitlets import List, Instance
-from IPython.utils.warn import warn, error
+from IPython.utils.warn import error
 
 #-----------------------------------------------------------------------------
 # Utilities
@@ -104,6 +102,70 b' class AliasError(Exception):'
 class InvalidAliasError(AliasError):
     pass
 
+class Alias(object):
+    """Callable object storing the details of one alias.
+
+    Instances are registered as magic functions to allow use of aliases.
+    """
+
+    # Prepare blacklist
+    blacklist = {'cd','popd','pushd','dhist','alias','unalias'}
+
+    def __init__(self, shell, name, cmd):
+        self.shell = shell
+        self.name = name
+        self.cmd = cmd
+        self.nargs = self.validate()
+
+    def validate(self):
+        """Validate the alias, and return the number of arguments."""
+        if self.name in self.blacklist:
+            raise InvalidAliasError("The name %s can't be aliased "
+                                    "because it is a keyword or builtin." % self.name)
+        try:
+            caller = self.shell.magics_manager.magics['line'][self.name]
+        except KeyError:
+            pass
+        else:
+            if not isinstance(caller, Alias):
+                raise InvalidAliasError("The name %s can't be aliased "
+                                        "because it is another magic command." % self.name)
+
+        if not (isinstance(self.cmd, basestring)):
+            raise InvalidAliasError("An alias command must be a string, "
+                                    "got: %r" % self.cmd)
+
+        nargs = self.cmd.count('%s')
+
+        if (nargs > 0) and (self.cmd.find('%l') >= 0):
+            raise InvalidAliasError('The %s and %l specifiers are mutually '
+                                    'exclusive in alias definitions.')
+
+        return nargs
+
+    def __repr__(self):
+        return "<alias {} for {!r}>".format(self.name, self.cmd)
+
+    def __call__(self, rest=''):
+        cmd = self.cmd
+        nargs = self.nargs
+        # Expand the %l special to be the user's input line
+        if cmd.find('%l') >= 0:
+            cmd = cmd.replace('%l', rest)
+            rest = ''
+        if nargs==0:
+            # Simple, argument-less aliases
+            cmd = '%s %s' % (cmd, rest)
+        else:
+            # Handle aliases with positional arguments
+            args = rest.split(None, nargs)
+            if len(args) < nargs:
+                raise UsageError('Alias <%s> requires %s arguments, %s given.' %
+                      (self.name, nargs, len(args)))
+            cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
+
+        self.shell.system(cmd)
+
 #-----------------------------------------------------------------------------
 # Main AliasManager class
 #-----------------------------------------------------------------------------
@@ -116,35 +178,19 b' class AliasManager(Configurable):'
 
     def __init__(self, shell=None, **kwargs):
         super(AliasManager, self).__init__(shell=shell, **kwargs)
-        self.alias_table = {}
-        self.exclude_aliases()
+        # For convenient access
+        self.linemagics = self.shell.magics_manager.magics['line']
         self.init_aliases()
 
-    def __contains__(self, name):
-        return name in self.alias_table
-
-    @property
-    def aliases(self):
-        return [(item[0], item[1][1]) for item in self.alias_table.iteritems()]
-
-    def exclude_aliases(self):
-        # set of things NOT to alias (keywords, builtins and some magics)
-        no_alias = set(['cd','popd','pushd','dhist','alias','unalias'])
-        no_alias.update(set(keyword.kwlist))
-        no_alias.update(set(__builtin__.__dict__.keys()))
-        self.no_alias = no_alias
-
     def init_aliases(self):
-        # Load default aliases
-        for name, cmd in self.default_aliases:
-            self.soft_define_alias(name, cmd)
-
-        # Load user aliases
-        for name, cmd in self.user_aliases:
+        # Load default & user aliases
+        for name, cmd in self.default_aliases + self.user_aliases:
             self.soft_define_alias(name, cmd)
 
-    def clear_aliases(self):
-        self.alias_table.clear()
+    @property
+    def aliases(self):
+        return [(n, func.cmd) for (n, func) in self.linemagics.items()
+                            if isinstance(func, Alias)]
 
     def soft_define_alias(self, name, cmd):
         """Define an alias, but don't raise on an AliasError."""
@@ -159,104 +205,33 b' class AliasManager(Configurable):'
         This will raise an :exc:`AliasError` if there are validation
         problems.
         """
-        nargs = self.validate_alias(name, cmd)
-        self.alias_table[name] = (nargs, cmd)
+        caller = Alias(shell=self.shell, name=name, cmd=cmd)
+        self.shell.magics_manager.register_function(caller, magic_kind='line',
+                                                    magic_name=name)
 
-    def undefine_alias(self, name):
-        if name in self.alias_table:
-            del self.alias_table[name]
+    def get_alias(self, name):
+        """Return an alias, or None if no alias by that name exists."""
+        aname = self.linemagics.get(name, None)
+        return aname if isinstance(aname, Alias) else None
 
-    def validate_alias(self, name, cmd):
-        """Validate an alias and return the its number of arguments."""
-        if name in self.no_alias:
-            raise InvalidAliasError("The name %s can't be aliased "
-                                    "because it is a keyword or builtin." % name)
-        if not (isinstance(cmd, basestring)):
-            raise InvalidAliasError("An alias command must be a string, "
-                                    "got: %r" % cmd)
-        nargs = cmd.count('%s')
-        if nargs>0 and cmd.find('%l')>=0:
-            raise InvalidAliasError('The %s and %l specifiers are mutually '
-                                    'exclusive in alias definitions.')
-        return nargs
+    def is_alias(self, name):
+        """Return whether or not a given name has been defined as an alias"""
+        return self.get_alias(name) is not None
 
-    def call_alias(self, alias, rest=''):
-        """Call an alias given its name and the rest of the line."""
-        cmd = self.transform_alias(alias, rest)
-        try:
-            self.shell.system(cmd)
-        except:
-            self.shell.showtraceback()
-
-    def transform_alias(self, alias,rest=''):
-        """Transform alias to system command string."""
-        nargs, cmd = self.alias_table[alias]
-
-        if ' ' in cmd and os.path.isfile(cmd):
-            cmd = '"%s"' % cmd
-
-        # Expand the %l special to be the user's input line
-        if cmd.find('%l') >= 0:
-            cmd = cmd.replace('%l', rest)
-            rest = ''
-        if nargs==0:
-            # Simple, argument-less aliases
-            cmd = '%s %s' % (cmd, rest)
+    def undefine_alias(self, name):
+        if self.is_alias(name):
+            del self.linemagics[name]
         else:
-            # Handle aliases with positional arguments
-            args = rest.split(None, nargs)
-            if len(args) < nargs:
-                raise AliasError('Alias <%s> requires %s arguments, %s given.' %
-                      (alias, nargs, len(args)))
-            cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:]))
-        return cmd
-
-    def expand_alias(self, line):
-        """ Expand an alias in the command line
+            raise ValueError('%s is not an alias' % name)
 
-        Returns the provided command line, possibly with the first word
-        (command) translated according to alias expansion rules.
-
-        [ipython]|16> _ip.expand_aliases("np myfile.txt")
-                 <16> 'q:/opt/np/notepad++.exe myfile.txt'
-        """
-
-        pre,_,fn,rest = split_user_input(line)
-        res = pre + self.expand_aliases(fn, rest)
-        return res
-
-    def expand_aliases(self, fn, rest):
-        """Expand multiple levels of aliases:
-
-        if:
-
-        alias foo bar /tmp
-        alias baz foo
-
-        then:
-
-        baz huhhahhei -> bar /tmp huhhahhei
-        """
-        line = fn + " " + rest
-
-        done = set()
-        while 1:
-            pre,_,fn,rest = split_user_input(line, shell_line_split)
-            if fn in self.alias_table:
-                if fn in done:
-                    warn("Cyclic alias definition, repeated '%s'" % fn)
-                    return ""
-                done.add(fn)
-
-                l2 = self.transform_alias(fn, rest)
-                if l2 == line:
-                    break
-                # ls -> ls -F should not recurse forever
-                if l2.split(None,1)[0] == line.split(None,1)[0]:
-                    line = l2
-                    break
-                line = l2
-            else:
-                break
-
-        return line
+    def clear_aliases(self):
+        for name, cmd in self.aliases:
+            self.undefine_alias(name)
+
+    def retrieve_alias(self, name):
+        """Retrieve the command to which an alias expands."""
+        caller = self.get_alias(name)
+        if caller:
+            return caller.cmd
+        else:
+            raise ValueError('%s is not an alias' % name)

IPython/core/application.py

@@ -53,6 +53,7 b' from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, Set, Instan'
 # aliases and flags
 
 base_aliases = {
+    'profile-dir' : 'ProfileDir.location',
     'profile' : 'BaseIPythonApplication.profile',
     'ipython-dir' : 'BaseIPythonApplication.ipython_dir',
     'log-level' : 'Application.log_level',

IPython/core/completer.py

@@ -24,26 +24,26 b' Tip: to use the tab key as the completion key, call'
 Notes:
 
 - Exceptions raised by the completer function are *ignored* (and
-generally cause the completion to fail).  This is a feature -- since
-readline sets the tty device in raw (or cbreak) mode, printing a
-traceback wouldn't work well without some complicated hoopla to save,
-reset and restore the tty state.
+  generally cause the completion to fail).  This is a feature -- since
+  readline sets the tty device in raw (or cbreak) mode, printing a
+  traceback wouldn't work well without some complicated hoopla to save,
+  reset and restore the tty state.
 
 - The evaluation of the NAME.NAME... form may cause arbitrary
-application defined code to be executed if an object with a
-__getattr__ hook is found.  Since it is the responsibility of the
-application (or the user) to enable this feature, I consider this an
-acceptable risk.  More complicated expressions (e.g. function calls or
-indexing operations) are *not* evaluated.
+  application defined code to be executed if an object with a
+  ``__getattr__`` hook is found.  Since it is the responsibility of the
+  application (or the user) to enable this feature, I consider this an
+  acceptable risk.  More complicated expressions (e.g. function calls or
+  indexing operations) are *not* evaluated.
 
 - GNU readline is also used by the built-in functions input() and
-raw_input(), and thus these also benefit/suffer from the completer
-features.  Clearly an interactive application can benefit by
-specifying its own completer function and using raw_input() for all
-its input.
+  raw_input(), and thus these also benefit/suffer from the completer
+  features.  Clearly an interactive application can benefit by
+  specifying its own completer function and using raw_input() for all
+  its input.
 
 - When the original stdin is not a tty device, GNU readline is never
-used, and this module (and the readline module) are silently inactive.
+  used, and this module (and the readline module) are silently inactive.
 """
 
 #*****************************************************************************
@@ -431,8 +431,7 b' class IPCompleter(Completer):'
     )
 
     def __init__(self, shell=None, namespace=None, global_namespace=None,
-                 alias_table=None, use_readline=True,
-                 config=None, **kwargs):
+                 use_readline=True, config=None, **kwargs):
         """IPCompleter() -> completer
 
         Return a completer object suitable for use by the readline library
@@ -441,17 +440,14 b' class IPCompleter(Completer):'
         Inputs:
 
         - shell: a pointer to the ipython shell itself.  This is needed
-        because this completer knows about magic functions, and those can
-        only be accessed via the ipython instance.
+          because this completer knows about magic functions, and those can
+          only be accessed via the ipython instance.
 
         - namespace: an optional dict where completions are performed.
 
         - global_namespace: secondary optional dict for completions, to
-        handle cases (such as IPython embedded inside functions) where
-        both Python scopes are visible.
-
-        - If alias_table is supplied, it should be a dictionary of aliases
-        to complete.
+          handle cases (such as IPython embedded inside functions) where
+          both Python scopes are visible.
 
         use_readline : bool, optional
           If true, use the readline library.  This completer can still function
@@ -476,9 +472,6 b' class IPCompleter(Completer):'
         # List where completion matches will be stored
         self.matches = []
         self.shell = shell
-        if alias_table is None:
-            alias_table = {}
-        self.alias_table = alias_table
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
@@ -505,7 +498,6 b' class IPCompleter(Completer):'
         self.matchers = [self.python_matches,
                          self.file_matches,
                          self.magic_matches,
-                         self.alias_matches,
                          self.python_func_kw_matches,
                          ]
 
@@ -628,22 +620,6 b' class IPCompleter(Completer):'
             comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
         return comp
 
-    def alias_matches(self, text):
-        """Match internal system aliases"""
-        #print 'Completer->alias_matches:',text,'lb',self.text_until_cursor # dbg
-
-        # if we are not in the first 'item', alias matching
-        # doesn't make sense - unless we are starting with 'sudo' command.
-        main_text = self.text_until_cursor.lstrip()
-        if ' ' in main_text and not main_text.startswith('sudo'):
-            return []
-        text = os.path.expanduser(text)
-        aliases =  self.alias_table.keys()
-        if text == '':
-            return aliases
-        else:
-            return [a for a in aliases if a.startswith(text)]
-
     def python_matches(self,text):
         """Match attributes or global python names"""
         

IPython/core/debugger.py

@@ -96,19 +96,24 b' class Tracer(object):'
     def __init__(self,colors=None):
         """Create a local debugger instance.
 
-        :Parameters:
+        Parameters
+        ----------
 
-          - `colors` (None): a string containing the name of the color scheme to
-        use, it must be one of IPython's valid color schemes.  If not given, the
-        function will default to the current IPython scheme when running inside
-        IPython, and to 'NoColor' otherwise.
+        colors : str, optional
+            The name of the color scheme to use, it must be one of IPython's
+            valid color schemes.  If not given, the function will default to
+            the current IPython scheme when running inside IPython, and to
+            'NoColor' otherwise.
 
-        Usage example:
+        Examples
+        --------
+        ::
 
-        from IPython.core.debugger import Tracer; debug_here = Tracer()
+            from IPython.core.debugger import Tracer; debug_here = Tracer()
 
-        ... later in your code
-        debug_here()  # -> will open up the debugger at that point.
+        Later in your code::
+        
+            debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation

IPython/core/display.py

@@ -508,9 +508,9 b' class Image(DisplayObject):'
     _ACCEPTABLE_EMBEDDINGS = [_FMT_JPEG, _FMT_PNG]
 
     def __init__(self, data=None, url=None, filename=None, format=u'png', embed=None, width=None, height=None, retina=False):
-        """Create a display an PNG/JPEG image given raw data.
+        """Create a PNG/JPEG image object given raw data.
 
-        When this object is returned by an expression or passed to the
+        When this object is returned by an input cell or passed to the
         display function, it will result in the image being displayed
         in the frontend.
 
@@ -657,35 +657,19 b' class Image(DisplayObject):'
         return unicode(s.split('.')[-1].lower())
 
 
-def clear_output(stdout=True, stderr=True, other=True):
+def clear_output(wait=False):
     """Clear the output of the current cell receiving output.
 
-    Optionally, each of stdout/stderr or other non-stream data (e.g. anything
-    produced by display()) can be excluded from the clear event.
-
-    By default, everything is cleared.
-
     Parameters
     ----------
-    stdout : bool [default: True]
-        Whether to clear stdout.
-    stderr : bool [default: True]
-        Whether to clear stderr.
-    other : bool [default: True]
-        Whether to clear everything else that is not stdout/stderr
-        (e.g. figures,images,HTML, any result of display()).
-    """
+    wait : bool [default: false]
+        Wait to clear the output until new output is available to replace it."""
     from IPython.core.interactiveshell import InteractiveShell
     if InteractiveShell.initialized():
-        InteractiveShell.instance().display_pub.clear_output(
-            stdout=stdout, stderr=stderr, other=other,
-        )
+        InteractiveShell.instance().display_pub.clear_output(wait)
     else:
         from IPython.utils import io
-        if stdout:
-            print('\033[2K\r', file=io.stdout, end='')
-            io.stdout.flush()
-        if stderr:
-            print('\033[2K\r', file=io.stderr, end='')
-            io.stderr.flush()
-
+        print('\033[2K\r', file=io.stdout, end='')
+        io.stdout.flush()
+        print('\033[2K\r', file=io.stderr, end='')
+        io.stderr.flush()

IPython/core/displaypub.py

@@ -108,14 +108,12 b' class DisplayPublisher(Configurable):'
         if 'text/plain' in data:
             print(data['text/plain'], file=io.stdout)
 
-    def clear_output(self, stdout=True, stderr=True, other=True):
+    def clear_output(self, wait=False):
         """Clear the output of the cell receiving output."""
-        if stdout:
-            print('\033[2K\r', file=io.stdout, end='')
-            io.stdout.flush()
-        if stderr:
-            print('\033[2K\r', file=io.stderr, end='')
-            io.stderr.flush()
+        print('\033[2K\r', file=io.stdout, end='')
+        io.stdout.flush()
+        print('\033[2K\r', file=io.stderr, end='')
+        io.stderr.flush()
 
 
 class CapturingDisplayPublisher(DisplayPublisher):
@@ -125,8 +123,8 b' class CapturingDisplayPublisher(DisplayPublisher):'
     def publish(self, source, data, metadata=None):
         self.outputs.append((source, data, metadata))
     
-    def clear_output(self, stdout=True, stderr=True, other=True):
-        super(CapturingDisplayPublisher, self).clear_output(stdout, stderr, other)
+    def clear_output(self, wait=False):
+        super(CapturingDisplayPublisher, self).clear_output(wait)
         if other:
             # empty the list, *do not* reassign a new list
             del self.outputs[:]

IPython/core/inputtransformer.py

@@ -278,6 +278,25 b' _help_end_re = re.compile(r"""(%{0,2}'
                               """,
                               re.VERBOSE)
 
+# Extra pseudotokens for multiline strings and data structures
+_MULTILINE_STRING = object()
+_MULTILINE_STRUCTURE = object()
+
+def _line_tokens(line):
+    """Helper for has_comment and ends_in_comment_or_string."""
+    readline = StringIO(line).readline
+    toktypes = set()
+    try:
+        for t in generate_tokens(readline):
+            toktypes.add(t[0])
+    except TokenError as e:
+        # There are only two cases where a TokenError is raised.
+        if 'multi-line string' in e.args[0]:
+            toktypes.add(_MULTILINE_STRING)
+        else:
+            toktypes.add(_MULTILINE_STRUCTURE)
+    return toktypes
+
 def has_comment(src):
     """Indicate whether an input line has (i.e. ends in, or is) a comment.
 
@@ -293,21 +312,31 b' def has_comment(src):'
     comment : bool
         True if source has a comment.
     """
-    readline = StringIO(src).readline
-    toktypes = set()
-    try:
-        for t in generate_tokens(readline):
-            toktypes.add(t[0])
-    except TokenError:
-        pass
-    return(tokenize2.COMMENT in toktypes)
+    return (tokenize2.COMMENT in _line_tokens(src))
 
+def ends_in_comment_or_string(src):
+    """Indicates whether or not an input line ends in a comment or within
+    a multiline string.
+    
+    Parameters
+    ----------
+    src : string
+      A single line input string.
+
+    Returns
+    -------
+    comment : bool
+        True if source ends in a comment or multiline string.
+    """
+    toktypes = _line_tokens(src)
+    return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
+        
 
 @StatelessInputTransformer.wrap
 def help_end(line):
     """Translate lines with ?/?? at the end"""
     m = _help_end_re.search(line)
-    if m is None or has_comment(line):
+    if m is None or ends_in_comment_or_string(line):
         return line
     target = m.group(1)
     esc = m.group(3)
@@ -359,8 +388,26 b' def cellmagic(end_on_blank_line=False):'
         line = tpl % (magic_name, first, u'\n'.join(body))
 
 
-def _strip_prompts(prompt_re):
-    """Remove matching input prompts from a block of input."""
+def _strip_prompts(prompt_re, initial_re=None):
+    """Remove matching input prompts from a block of input.
+    
+    Parameters
+    ----------
+    prompt_re : regular expression
+        A regular expression matching any input prompt (including continuation)
+    initial_re : regular expression, optional
+        A regular expression matching only the initial prompt, but not continuation.
+        If no initial expression is given, prompt_re will be used everywhere.
+        Used mainly for plain Python prompts, where the continuation prompt
+        ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
+    
+    If initial_re and prompt_re differ,
+    only initial_re will be tested against the first line.
+    If any prompt is found on the first two lines,
+    prompts will be stripped from the rest of the block.
+    """
+    if initial_re is None:
+        initial_re = prompt_re
     line = ''
     while True:
         line = (yield line)
@@ -368,18 +415,19 b' def _strip_prompts(prompt_re):'
         # First line of cell
         if line is None:
             continue
-        out, n1 = prompt_re.subn('', line, count=1)
+        out, n1 = initial_re.subn('', line, count=1)
         line = (yield out)
         
-        # Second line of cell, because people often copy from just after the
-        # first prompt, so we might not see it in the first line.
         if line is None:
             continue
+        # check for any prompt on the second line of the cell,
+        # because people often copy from just after the first prompt,
+        # so we might not see it in the first line.
         out, n2 = prompt_re.subn('', line, count=1)
         line = (yield out)
         
         if n1 or n2:
-            # Found the input prompt in the first two lines - check for it in
+            # Found a prompt in the first two lines - check for it in
             # the rest of the cell as well.
             while line is not None:
                 line = (yield prompt_re.sub('', line, count=1))
@@ -394,7 +442,8 b' def classic_prompt():'
     """Strip the >>>/... prompts of the Python interactive shell."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)')
-    return _strip_prompts(prompt_re)
+    initial_re = re.compile(r'^(>>> ?)')
+    return _strip_prompts(prompt_re, initial_re)
 
 @CoroutineInputTransformer.wrap
 def ipy_prompt():

IPython/core/interactiveshell.py

@@ -47,7 +47,6 b' from IPython.core.displayhook import DisplayHook'
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import UsageError
 from IPython.core.extensions import ExtensionManager
-from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.inputsplitter import IPythonInputSplitter, ESC_MAGIC, ESC_MAGIC2
@@ -186,6 +185,13 b' class ReadlineNoRecord(object):'
         ghi = self.shell.readline.get_history_item
         return [ghi(x) for x in range(start, end)]
 
+
+@undoc
+class DummyMod(object):
+    """A dummy module used for IPython's interactive module when
+    a namespace must be assigned to the module's __dict__."""
+    pass
+
 #-----------------------------------------------------------------------------
 # Main IPython class
 #-----------------------------------------------------------------------------
@@ -464,7 +470,6 b' class InteractiveShell(SingletonConfigurable):'
         # because it and init_io have to come after init_readline.
         self.init_user_ns()
         self.init_logger()
-        self.init_alias()
         self.init_builtins()
 
         # The following was in post_config_initialization
@@ -496,6 +501,7 b' class InteractiveShell(SingletonConfigurable):'
         self.init_displayhook()
         self.init_latextool()
         self.init_magics()
+        self.init_alias()
         self.init_logstart()
         self.init_pdb()
         self.init_extension_manager()
@@ -819,15 +825,18 b' class InteractiveShell(SingletonConfigurable):'
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
-    def new_main_mod(self, filename):
+    def new_main_mod(self, filename, modname):
         """Return a new 'main' module object for user code execution.
         
         ``filename`` should be the path of the script which will be run in the
         module. Requests with the same filename will get the same module, with
         its namespace cleared.
         
+        ``modname`` should be the module name - normally either '__main__' or
+        the basename of the file without the extension.
+        
         When scripts are executed via %run, we must keep a reference to their
-        __main__ module (a FakeModule instance) around so that Python doesn't
+        __main__ module around so that Python doesn't
         clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
@@ -840,9 +849,16 b' class InteractiveShell(SingletonConfigurable):'
         try:
             main_mod = self._main_mod_cache[filename]
         except KeyError:
-            main_mod = self._main_mod_cache[filename] = FakeModule()
+            main_mod = self._main_mod_cache[filename] = types.ModuleType(modname,
+                        doc="Module created for script run in IPython")
         else:
-            init_fakemod_dict(main_mod)
+            main_mod.__dict__.clear()
+            main_mod.__name__ = modname
+        
+        main_mod.__file__ = filename
+        # It seems pydoc (and perhaps others) needs any module instance to
+        # implement a __nonzero__ method
+        main_mod.__nonzero__ = lambda : True
         
         return main_mod
 
@@ -856,7 +872,7 b' class InteractiveShell(SingletonConfigurable):'
 
         In [15]: import IPython
 
-        In [16]: m = _ip.new_main_mod(IPython.__file__)
+        In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
 
         In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
@@ -900,9 +916,9 b' class InteractiveShell(SingletonConfigurable):'
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
-          flag and does not actually invoke the debugger if the flag is false.
-          The 'force' option forces the debugger to activate even if the flag
-          is false.
+            flag and does not actually invoke the debugger if the flag is false.
+            The 'force' option forces the debugger to activate even if the flag
+            is false.
         """
 
         if not (force or self.call_pdb):
@@ -970,7 +986,7 b' class InteractiveShell(SingletonConfigurable):'
 
         # A record of hidden variables we have added to the user namespace, so
         # we can list later only variables defined in actual interactive use.
-        self.user_ns_hidden = set()
+        self.user_ns_hidden = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
@@ -1035,9 +1051,6 b' class InteractiveShell(SingletonConfigurable):'
         """
         if user_module is None and user_ns is not None:
             user_ns.setdefault("__name__", "__main__")
-            class DummyMod(object):
-                "A dummy module used for IPython's interactive namespace."
-                pass
             user_module = DummyMod()
             user_module.__dict__ = user_ns
             
@@ -1150,7 +1163,7 b' class InteractiveShell(SingletonConfigurable):'
         
         Note that this does not include the displayhook, which also caches
         objects from the output."""
-        return [self.user_ns, self.user_global_ns] + \
+        return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
                [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True):
@@ -1301,7 +1314,8 b' class InteractiveShell(SingletonConfigurable):'
         # And configure interactive visibility
         user_ns_hidden = self.user_ns_hidden
         if interactive:
-            user_ns_hidden.difference_update(vdict)
+            for name in vdict:
+                user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
     
@@ -1321,7 +1335,7 b' class InteractiveShell(SingletonConfigurable):'
         for name, obj in variables.iteritems():
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
-                self.user_ns_hidden.discard(name)
+                self.user_ns_hidden.pop(name, None)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
@@ -1349,9 +1363,7 b' class InteractiveShell(SingletonConfigurable):'
             namespaces = [ ('Interactive', self.user_ns),
                            ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
-                           ('Alias', self.alias_manager.alias_table),
                            ]
-            alias_ns = self.alias_manager.alias_table
 
         # initialize results to 'null'
         found = False; obj = None;  ospace = None;  ds = None;
@@ -1390,8 +1402,6 b' class InteractiveShell(SingletonConfigurable):'
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
-                    if ns == alias_ns:
-                        isalias = True
                     break  # namespace loop
 
         # Try to see if it's magic
@@ -1926,7 +1936,6 b' class InteractiveShell(SingletonConfigurable):'
         self.Completer = IPCompleter(shell=self,
                                      namespace=self.user_ns,
                                      global_namespace=self.user_global_ns,
-                                     alias_table=self.alias_manager.alias_table,
                                      use_readline=self.has_readline,
                                      parent=self,
                                      )
@@ -2291,7 +2300,6 b' class InteractiveShell(SingletonConfigurable):'
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, parent=self)
         self.configurables.append(self.alias_manager)
-        self.ns_table['alias'] = self.alias_manager.alias_table,
 
     #-------------------------------------------------------------------------
     # Things related to extensions
@@ -2973,7 +2981,7 b' class InteractiveShell(SingletonConfigurable):'
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
-          immediately, and the file is closed again."""
+            immediately, and the file is closed again."""
 
         filename = tempfile.mktemp('.py', prefix)
         self.tempfiles.append(filename)
@@ -3016,13 +3024,14 b' class InteractiveShell(SingletonConfigurable):'
 
         Optional Parameters:
           - raw(False): by default, the processed input is used.  If this is
-          true, the raw input history is used instead.
+            true, the raw input history is used instead.
 
         Note that slices can be called with two notations:
 
         N:M -> standard python form, means including items N...(M-1).
 
-        N-M -> include items N..M (closed endpoint)."""
+        N-M -> include items N..M (closed endpoint).
+        """
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 

IPython/core/logger.py

@@ -171,11 +171,11 b' which already exists. But you must first start the logging process with'
         Inputs:
 
         - line_mod: possibly modified input, such as the transformations made
-        by input prefilters or input handlers of various kinds.  This should
-        always be valid Python.
+          by input prefilters or input handlers of various kinds. This should
+          always be valid Python.
 
-        - line_ori: unmodified input line from the user.  This is not
-        necessarily valid Python.
+        - line_ori: unmodified input line from the user. This is not
+          necessarily valid Python.
         """
 
         # Write the log line, but decide which one according to the

IPython/core/magic.py

@@ -489,11 +489,11 b' class Magics(object):'
     MUST:
 
     - Use the method decorators `@line_magic` and `@cell_magic` to decorate
-    individual methods as magic functions, AND
+      individual methods as magic functions, AND
 
     - Use the class decorator `@magics_class` to ensure that the magic
-    methods are properly registered at the instance level upon instance
-    initialization.
+      methods are properly registered at the instance level upon instance
+      initialization.
 
     See :mod:`magic_functions` for examples of actual implementation classes.
     """

IPython/core/magic_arguments.py

@@ -50,9 +50,9 b' Inheritance diagram:'
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
+import argparse
 
 # Our own imports
-from IPython.external import argparse
 from IPython.core.error import UsageError
 from IPython.utils.process import arg_split
 from IPython.utils.text import dedent

IPython/core/magics/execution.py

@@ -56,6 +56,37 b' from IPython.utils.warn import warn, error'
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
+
+class TimeitResult(object):
+    """
+    Object returned by the timeit magic with info about the run.
+
+    Contain the following attributes :
+
+    loops: (int) number of loop done per measurement
+    repeat: (int) number of time the mesurement has been repeated
+    best: (float) best execusion time / number
+    all_runs: (list of float) execusion time of each run (in s)
+    compile_time: (float) time of statement compilation (s)
+
+    """
+
+    def __init__(self, loops, repeat, best, all_runs, compile_time, precision):
+        self.loops = loops
+        self.repeat = repeat
+        self.best = best
+        self.all_runs = all_runs
+        self.compile_time = compile_time
+        self._precision = precision
+
+    def _repr_pretty_(self, p , cycle):
+         unic =  u"%d loops, best of %d: %s per loop" % (self.loops, self.repeat,
+                                            _format_time(self.best, self._precision))
+         p.text(u'<TimeitResult : '+unic+u'>')
+
+
+
+
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc.
@@ -102,75 +133,84 b' python-profiler package from non-free.""")'
 
         Options:
 
-        -l <limit>: you can place restrictions on what or how much of the
-        profile gets printed. The limit value can be:
-
-          * A string: only information for function names containing this string
-          is printed.
-
-          * An integer: only these many lines are printed.
-
-          * A float (between 0 and 1): this fraction of the report is printed
-          (for example, use a limit of 0.4 to see the topmost 40% only).
-
-        You can combine several limits with repeated use of the option. For
-        example, '-l __init__ -l 5' will print only the topmost 5 lines of
-        information about class constructors.
-
-        -r: return the pstats.Stats object generated by the profiling. This
-        object has all the information about the profile in it, and you can
-        later use it for further analysis or in other functions.
-
-       -s <key>: sort profile by given key. You can provide more than one key
-        by using the option several times: '-s key1 -s key2 -s key3...'. The
-        default sorting key is 'time'.
-
-        The following is copied verbatim from the profile documentation
-        referenced below:
-
-        When more than one key is provided, additional keys are used as
-        secondary criteria when the there is equality in all keys selected
-        before them.
-
-        Abbreviations can be used for any key names, as long as the
-        abbreviation is unambiguous.  The following are the keys currently
-        defined:
-
-                Valid Arg       Meaning
-                  "calls"      call count
-                  "cumulative" cumulative time
-                  "file"       file name
-                  "module"     file name
-                  "pcalls"     primitive call count
-                  "line"       line number
-                  "name"       function name
-                  "nfl"        name/file/line
-                  "stdname"    standard name
-                  "time"       internal time
-
-        Note that all sorts on statistics are in descending order (placing
-        most time consuming items first), where as name, file, and line number
-        searches are in ascending order (i.e., alphabetical). The subtle
-        distinction between "nfl" and "stdname" is that the standard name is a
-        sort of the name as printed, which means that the embedded line
-        numbers get compared in an odd way.  For example, lines 3, 20, and 40
-        would (if the file names were the same) appear in the string order
-        "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
-        line numbers.  In fact, sort_stats("nfl") is the same as
-        sort_stats("name", "file", "line").
-
-        -T <filename>: save profile results as shown on screen to a text
-        file. The profile is still shown on screen.
-
-        -D <filename>: save (via dump_stats) profile statistics to given
-        filename. This data is in a format understood by the pstats module, and
-        is generated by a call to the dump_stats() method of profile
-        objects. The profile is still shown on screen.
-
-        -q: suppress output to the pager.  Best used with -T and/or -D above.
+        -l <limit>
+          you can place restrictions on what or how much of the
+          profile gets printed. The limit value can be:
+
+             * A string: only information for function names containing this string
+               is printed.
+
+             * An integer: only these many lines are printed.
+
+             * A float (between 0 and 1): this fraction of the report is printed
+               (for example, use a limit of 0.4 to see the topmost 40% only).
+
+          You can combine several limits with repeated use of the option. For
+          example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
+          information about class constructors.
+
+        -r
+          return the pstats.Stats object generated by the profiling. This
+          object has all the information about the profile in it, and you can
+          later use it for further analysis or in other functions.
+
+        -s <key>
+          sort profile by given key. You can provide more than one key
+          by using the option several times: '-s key1 -s key2 -s key3...'. The
+          default sorting key is 'time'.
+
+          The following is copied verbatim from the profile documentation
+          referenced below:
+
+          When more than one key is provided, additional keys are used as
+          secondary criteria when the there is equality in all keys selected
+          before them.
+
+          Abbreviations can be used for any key names, as long as the
+          abbreviation is unambiguous.  The following are the keys currently
+          defined:
+
+          ============  =====================
+          Valid Arg     Meaning
+          ============  =====================
+          "calls"       call count
+          "cumulative"  cumulative time
+          "file"        file name
+          "module"      file name
+          "pcalls"      primitive call count
+          "line"        line number
+          "name"        function name
+          "nfl"         name/file/line
+          "stdname"     standard name
+          "time"        internal time
+          ============  =====================
+
+          Note that all sorts on statistics are in descending order (placing
+          most time consuming items first), where as name, file, and line number
+          searches are in ascending order (i.e., alphabetical). The subtle
+          distinction between "nfl" and "stdname" is that the standard name is a
+          sort of the name as printed, which means that the embedded line
+          numbers get compared in an odd way.  For example, lines 3, 20, and 40
+          would (if the file names were the same) appear in the string order
+          "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
+          line numbers.  In fact, sort_stats("nfl") is the same as
+          sort_stats("name", "file", "line").
+
+        -T <filename>
+          save profile results as shown on screen to a text
+          file. The profile is still shown on screen.
+
+        -D <filename>
+          save (via dump_stats) profile statistics to given
+          filename. This data is in a format understood by the pstats module, and
+          is generated by a call to the dump_stats() method of profile
+          objects. The profile is still shown on screen.
+
+        -q
+          suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
-        '%run -p [prof_opts] filename.py [args to program]' where prof_opts
+        ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
@@ -362,7 +402,8 b' python-profiler package from non-free.""")'
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
-        Usage:
+        Usage::
+        
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | file ) [args]
@@ -371,14 +412,13 b' python-profiler package from non-free.""")'
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
-        This is similar to running at a system prompt:\\
-          $ python file args\\
+        This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
-        __name__=='__main__' and sys.argv constructed as indicated. It thus
+        ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
@@ -390,33 +430,37 b' python-profiler package from non-free.""")'
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
-        *two* back slashes (e.g., '\\\\*') to suppress expansions.
+        *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         Options:
 
-        -n: __name__ is NOT set to '__main__', but to the running file's name
-        without extension (as python does under import).  This allows running
-        scripts and reloading the definitions in them without calling code
-        protected by an ' if __name__ == "__main__" ' clause.
-
-        -i: run the file in IPython's namespace instead of an empty one. This
-        is useful if you are experimenting with code written in a text editor
-        which depends on variables defined interactively.
-
-        -e: ignore sys.exit() calls or SystemExit exceptions in the script
-        being run.  This is particularly useful if IPython is being used to
-        run unittests, which always exit with a sys.exit() call.  In such
-        cases you are interested in the output of the test results, not in
-        seeing a traceback of the unittest module.
-
-        -t: print timing information at the end of the run.  IPython will give
-        you an estimated CPU time consumption for your script, which under
-        Unix uses the resource module to avoid the wraparound problems of
-        time.clock().  Under Unix, an estimate of time spent on system tasks
-        is also given (for Windows platforms this is reported as 0.0).
-
-        If -t is given, an additional -N<N> option can be given, where <N>
+        -n
+          __name__ is NOT set to '__main__', but to the running file's name
+          without extension (as python does under import).  This allows running
+          scripts and reloading the definitions in them without calling code
+          protected by an ``if __name__ == "__main__"`` clause.
+
+        -i
+          run the file in IPython's namespace instead of an empty one. This
+          is useful if you are experimenting with code written in a text editor
+          which depends on variables defined interactively.
+
+        -e
+          ignore sys.exit() calls or SystemExit exceptions in the script
+          being run.  This is particularly useful if IPython is being used to
+          run unittests, which always exit with a sys.exit() call.  In such
+          cases you are interested in the output of the test results, not in
+          seeing a traceback of the unittest module.
+
+        -t
+          print timing information at the end of the run.  IPython will give
+          you an estimated CPU time consumption for your script, which under
+          Unix uses the resource module to avoid the wraparound problems of
+          time.clock().  Under Unix, an estimate of time spent on system tasks
+          is also given (for Windows platforms this is reported as 0.0).
+
+        If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
@@ -424,74 +468,78 b' python-profiler package from non-free.""")'
 
             In [1]: run -t uniq_stable
 
-            IPython CPU timings (estimated):\\
-              User  :    0.19597 s.\\
-              System:        0.0 s.\\
+            IPython CPU timings (estimated):
+              User  :    0.19597 s.
+              System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
-            IPython CPU timings (estimated):\\
-            Total runs performed: 5\\
-              Times :      Total       Per run\\
-              User  :   0.910862 s,  0.1821724 s.\\
+            IPython CPU timings (estimated):
+            Total runs performed: 5
+              Times :      Total       Per run
+              User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
-        -d: run your program under the control of pdb, the Python debugger.
-        This allows you to execute your program step by step, watch variables,
-        etc.  Internally, what IPython does is similar to calling:
+        -d
+          run your program under the control of pdb, the Python debugger.
+          This allows you to execute your program step by step, watch variables,
+          etc.  Internally, what IPython does is similar to calling::
 
-          pdb.run('execfile("YOURFILENAME")')
+              pdb.run('execfile("YOURFILENAME")')
 
-        with a breakpoint set on line 1 of your file.  You can change the line
-        number for this automatic breakpoint to be <N> by using the -bN option
-        (where N must be an integer).  For example::
+          with a breakpoint set on line 1 of your file.  You can change the line
+          number for this automatic breakpoint to be <N> by using the -bN option
+          (where N must be an integer). For example::
 
-          %run -d -b40 myscript
+              %run -d -b40 myscript
 
-        will set the first breakpoint at line 40 in myscript.py.  Note that
-        the first breakpoint must be set on a line which actually does
-        something (not a comment or docstring) for it to stop execution.
+          will set the first breakpoint at line 40 in myscript.py.  Note that
+          the first breakpoint must be set on a line which actually does
+          something (not a comment or docstring) for it to stop execution.
 
-        Or you can specify a breakpoint in a different file::
+          Or you can specify a breakpoint in a different file::
 
-          %run -d -b myotherfile.py:20 myscript
+              %run -d -b myotherfile.py:20 myscript
 
-        When the pdb debugger starts, you will see a (Pdb) prompt.  You must
-        first enter 'c' (without quotes) to start execution up to the first
-        breakpoint.
+          When the pdb debugger starts, you will see a (Pdb) prompt.  You must
+          first enter 'c' (without quotes) to start execution up to the first
+          breakpoint.
 
-        Entering 'help' gives information about the use of the debugger.  You
-        can easily see pdb's full documentation with "import pdb;pdb.help()"
-        at a prompt.
+          Entering 'help' gives information about the use of the debugger.  You
+          can easily see pdb's full documentation with "import pdb;pdb.help()"
+          at a prompt.
 
-        -p: run program under the control of the Python profiler module (which
-        prints a detailed report of execution times, function calls, etc).
+        -p
+          run program under the control of the Python profiler module (which
+          prints a detailed report of execution times, function calls, etc).
 
-        You can pass other options after -p which affect the behavior of the
-        profiler itself. See the docs for %prun for details.
+          You can pass other options after -p which affect the behavior of the
+          profiler itself. See the docs for %prun for details.
 
-        In this mode, the program's variables do NOT propagate back to the
-        IPython interactive namespace (because they remain in the namespace
-        where the profiler executes them).
+          In this mode, the program's variables do NOT propagate back to the
+          IPython interactive namespace (because they remain in the namespace
+          where the profiler executes them).
 
-        Internally this triggers a call to %prun, see its documentation for
-        details on the options available specifically for profiling.
+          Internally this triggers a call to %prun, see its documentation for
+          details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy, the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
-        -m: specify module name to load instead of script path. Similar to
-        the -m option for the python interpreter. Use this option last if you
-        want to combine with other %run options. Unlike the python interpreter
-        only source modules are allowed no .pyc or .pyo files.
-        For example::
+        -m
+          specify module name to load instead of script path. Similar to
+          the -m option for the python interpreter. Use this option last if you
+          want to combine with other %run options. Unlike the python interpreter
+          only source modules are allowed no .pyc or .pyo files.
+          For example::
 
-            %run -m example
+              %run -m example
 
-        will run the example module.
+          will run the example module.
 
-        -G: disable shell-like glob expansion of arguments.
+        -G
+          disable shell-like glob expansion of arguments.
 
         """
 
@@ -550,6 +598,11 b' python-profiler package from non-free.""")'
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.user_module
+            
+            # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
+            # set the __file__ global in the script's namespace
+            # TK: Is this necessary in interactive mode?
+            prog_ns['__file__'] = filename
         else:
             # Run in a fresh, empty namespace
             if 'n' in opts:
@@ -560,13 +613,8 b' python-profiler package from non-free.""")'
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
-            main_mod = self.shell.new_main_mod(filename)
+            main_mod = self.shell.new_main_mod(filename, name)
             prog_ns = main_mod.__dict__
-            prog_ns['__name__'] = name
-
-        # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
-        # set the __file__ global in the script's namespace
-        prog_ns['__file__'] = filename
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
@@ -786,9 +834,9 b' python-profiler package from non-free.""")'
         """Time execution of a Python statement or expression
 
         Usage, in line mode:
-          %timeit [-n<N> -r<R> [-t|-c]] statement
+          %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
         or in cell mode:
-          %%timeit [-n<N> -r<R> [-t|-c]] setup_code
+          %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
           code
           code...
 
@@ -819,6 +867,11 b' python-profiler package from non-free.""")'
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
+        -q: Quiet, do not print result.
+
+        -o: return a TimeitResult that can be stored in a variable to inspect
+            the result in more details.
+
 
         Examples
         --------
@@ -851,7 +904,7 b' python-profiler package from non-free.""")'
 
         import timeit
 
-        opts, stmt = self.parse_options(line,'n:r:tcp:',
+        opts, stmt = self.parse_options(line,'n:r:tcp:qo',
                                         posix=False, strict=False)
         if stmt == "" and cell is None:
             return
@@ -860,6 +913,8 b' python-profiler package from non-free.""")'
         number = int(getattr(opts, "n", 0))
         repeat = int(getattr(opts, "r", timeit.default_repeat))
         precision = int(getattr(opts, "p", 3))
+        quiet = 'q' in opts
+        return_result = 'o' in opts
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
@@ -931,13 +986,15 b' python-profiler package from non-free.""")'
                 if timer.timeit(number) >= 0.2:
                     break
                 number *= 10
-
-        best = min(timer.repeat(repeat, number)) / number
-
-        print u"%d loops, best of %d: %s per loop" % (number, repeat,
-                                                          _format_time(best, precision))
-        if tc > tc_min:
-            print "Compiler time: %.2f s" % tc
+        all_runs = timer.repeat(repeat, number)
+        best = min(all_runs) / number
+        if not quiet :
+            print u"%d loops, best of %d: %s per loop" % (number, repeat,
+                                                              _format_time(best, precision))
+            if tc > tc_min:
+                print "Compiler time: %.2f s" % tc
+        if return_result:
+            return TimeitResult(number, repeat, best, all_runs, tc, precision)
 
     @skip_doctest
     @needs_local_scope

IPython/core/magics/logging.py

@@ -42,34 +42,48 b' class LoggingMagics(Magics):'
         history up to that point and then continues logging.
 
         %logstart takes a second optional parameter: logging mode. This can be one
-        of (note that the modes are given unquoted):\\
-          append: well, that says it.\\
-          backup: rename (if exists) to name~ and start name.\\
-          global: single logfile in your home dir, appended to.\\
-          over  : overwrite existing log.\\
-          rotate: create rotating logs name.1~, name.2~, etc.
+        of (note that the modes are given unquoted):
+
+        append
+            Keep logging at the end of any existing file.
+
+        backup
+            Rename any existing file to name~ and start name.
+
+        global
+            Append to  a single logfile in your home directory.
+
+        over
+            Overwrite any existing log.
+
+        rotate
+            Create rotating logs: name.1~, name.2~, etc.
 
         Options:
 
-          -o: log also IPython's output.  In this mode, all commands which
-          generate an Out[NN] prompt are recorded to the logfile, right after
-          their corresponding input line.  The output lines are always
-          prepended with a '#[Out]# ' marker, so that the log remains valid
-          Python code.
+          -o
+            log also IPython's output. In this mode, all commands which
+            generate an Out[NN] prompt are recorded to the logfile, right after
+            their corresponding input line. The output lines are always
+            prepended with a '#[Out]# ' marker, so that the log remains valid
+            Python code.
 
           Since this marker is always the same, filtering only the output from
           a log is very easy, using for example a simple awk call::
 
             awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py
 
-          -r: log 'raw' input.  Normally, IPython's logs contain the processed
-          input, so that user lines are logged in their final form, converted
-          into valid Python.  For example, %Exit is logged as
-          _ip.magic("Exit").  If the -r flag is given, all input is logged
-          exactly as typed, with no transformations applied.
-
-          -t: put timestamps before each input line logged (these are put in
-          comments)."""
+          -r
+            log 'raw' input.  Normally, IPython's logs contain the processed
+            input, so that user lines are logged in their final form, converted
+            into valid Python.  For example, %Exit is logged as
+            _ip.magic("Exit").  If the -r flag is given, all input is logged
+            exactly as typed, with no transformations applied.
+
+          -t
+            put timestamps before each input line logged (these are put in
+            comments).
+        """
 
         opts,par = self.parse_options(parameter_s,'ort')
         log_output = 'o' in opts

IPython/core/magics/namespace.py

@@ -265,9 +265,10 b' class NamespaceMagics(Magics):'
 
         user_ns = self.shell.user_ns
         user_ns_hidden = self.shell.user_ns_hidden
+        nonmatching = object()  # This can never be in user_ns
         out = [ i for i in user_ns
                 if not i.startswith('_') \
-                and not i in user_ns_hidden ]
+                and (user_ns[i] is not user_ns_hidden.get(i, nonmatching)) ]
 
         typelist = parameter_s.split()
         if typelist:
@@ -353,10 +354,10 b' class NamespaceMagics(Magics):'
           - For {},[],(): their length.
 
           - For numpy arrays, a summary with shape, number of
-          elements, typecode and size in memory.
+            elements, typecode and size in memory.
 
           - Everything else: a string representation, snipping their middle if
-          too long.
+            too long.
 
         Examples
         --------

IPython/core/magics/osm.py

@@ -26,6 +26,7 b' from pprint import pformat'
 from IPython.core import magic_arguments
 from IPython.core import oinspect
 from IPython.core import page
+from IPython.core.alias import AliasError, Alias
 from IPython.core.error import UsageError
 from IPython.core.magic import  (
     Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic
@@ -113,10 +114,14 b' class OSMagics(Magics):'
         # Now try to define a new one
         try:
             alias,cmd = par.split(None, 1)
-        except:
-            print oinspect.getdoc(self.alias)
-        else:
-            self.shell.alias_manager.soft_define_alias(alias, cmd)
+        except TypeError:
+            print(oinspect.getdoc(self.alias))
+            return
+        
+        try:
+            self.shell.alias_manager.define_alias(alias, cmd)
+        except AliasError as e:
+            print(e)
     # end magic_alias
 
     @line_magic
@@ -124,7 +129,12 b' class OSMagics(Magics):'
         """Remove an alias"""
 
         aname = parameter_s.strip()
-        self.shell.alias_manager.undefine_alias(aname)
+        try:
+            self.shell.alias_manager.undefine_alias(aname)
+        except ValueError as e:
+            print(e)
+            return
+        
         stored = self.shell.db.get('stored_aliases', {} )
         if aname in stored:
             print "Removing %stored alias",aname
@@ -182,7 +192,7 b' class OSMagics(Magics):'
                             try:
                                 # Removes dots from the name since ipython
                                 # will assume names with dots to be python.
-                                if ff not in self.shell.alias_manager:
+                                if not self.shell.alias_manager.is_alias(ff):
                                     self.shell.alias_manager.define_alias(
                                         ff.replace('.',''), ff)
                             except InvalidAliasError:
@@ -190,7 +200,7 b' class OSMagics(Magics):'
                             else:
                                 syscmdlist.append(ff)
             else:
-                no_alias = self.shell.alias_manager.no_alias
+                no_alias = Alias.blacklist
                 for pdir in path:
                     os.chdir(pdir)
                     for ff in os.listdir(pdir):

IPython/core/oinspect.py

@@ -156,8 +156,8 b' def getsource(obj,is_binary=False):'
     Optional inputs:
 
     - is_binary: whether the object is known to come from a binary source.
-    This implementation will skip returning any output for binary objects, but
-    custom extractors may know how to meaningfully process them."""
+      This implementation will skip returning any output for binary objects, but
+      custom extractors may know how to meaningfully process them."""
 
     if is_binary:
         return None
@@ -545,7 +545,7 b' class Inspector:'
         - formatter: special formatter for docstrings (see pdoc)
 
         - info: a structure with some information fields which may have been
-        precomputed already.
+          precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
@@ -609,7 +609,7 b' class Inspector:'
         - formatter: special formatter for docstrings (see pdoc)
 
         - info: a structure with some information fields which may have been
-        precomputed already.
+          precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
@@ -829,8 +829,8 b' class Inspector:'
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
-        searches and optionally a type specification to narrow the search to
-        objects of that type.
+          searches and optionally a type specification to narrow the search to
+          objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
@@ -841,7 +841,7 b' class Inspector:'
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
-          underscores.
+            underscores.
         """
         #print 'ps pattern:<%r>' % pattern # dbg
 

IPython/core/page.py

@@ -325,9 +325,11 b" def snip_print(str,width = 75,print_full = 0,header = ''):"
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
+    
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
+    
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:

IPython/core/prefilter.py

@@ -488,22 +488,6 b' class AutoMagicChecker(PrefilterChecker):'
         return self.prefilter_manager.get_handler_by_name('magic')
 
 
-class AliasChecker(PrefilterChecker):
-
-    priority = Integer(800, config=True)
-
-    def check(self, line_info):
-        "Check if the initital identifier on the line is an alias."
-        # Note: aliases can not contain '.'
-        head = line_info.ifun.split('.',1)[0]
-        if line_info.ifun not in self.shell.alias_manager \
-               or head not in self.shell.alias_manager \
-               or is_shadowed(head, self.shell):
-            return None
-
-        return self.prefilter_manager.get_handler_by_name('alias')
-
-
 class PythonOpsChecker(PrefilterChecker):
 
     priority = Integer(900, config=True)
@@ -591,20 +575,6 b' class PrefilterHandler(Configurable):'
         return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name)
 
 
-class AliasHandler(PrefilterHandler):
-
-    handler_name = Unicode('alias')
-
-    def handle(self, line_info):
-        """Handle alias input lines. """
-        transformed = self.shell.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest)
-        # pre is needed, because it carries the leading whitespace.  Otherwise
-        # aliases won't work in indented sections.
-        line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, transformed)
-
-        return line_out
-
-
 class MacroHandler(PrefilterHandler):
     handler_name = Unicode("macro")
 
@@ -730,14 +700,12 b' _default_checkers = ['
     IPyAutocallChecker,
     AssignmentChecker,
     AutoMagicChecker,
-    AliasChecker,
     PythonOpsChecker,
     AutocallChecker
 ]
 
 _default_handlers = [
     PrefilterHandler,
-    AliasHandler,
     MacroHandler,
     MagicHandler,
     AutoHandler,

IPython/core/profileapp.py

@@ -11,7 +11,7 b' Authors:'
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -28,6 +28,7 b' from IPython.core.application import ('
     BaseIPythonApplication, base_flags
 )
 from IPython.core.profiledir import ProfileDir
+from IPython.utils.importstring import import_item
 from IPython.utils.path import get_ipython_dir, get_ipython_package_dir
 from IPython.utils.traitlets import Unicode, Bool, Dict
 
@@ -206,6 +207,8 b' class ProfileCreate(BaseIPythonApplication):'
     description = create_help
     examples = _create_examples
     auto_create = Bool(True, config=False)
+    def _log_format_default(self):
+        return "[%(name)s] %(message)s"
 
     def _copy_config_files_default(self):
         return True
@@ -234,31 +237,32 b' class ProfileCreate(BaseIPythonApplication):'
     flags = Dict(create_flags)
 
     classes = [ProfileDir]
+    
+    def _import_app(self, app_path):
+        """import an app class"""
+        app = None
+        name = app_path.rsplit('.', 1)[-1]
+        try:
+            app = import_item(app_path)
+        except ImportError as e:
+            self.log.info("Couldn't import %s, config file will be excluded", name)
+        except Exception:
+            self.log.warn('Unexpected error importing %s', name, exc_info=True)
+        return app
 
     def init_config_files(self):
         super(ProfileCreate, self).init_config_files()
         # use local imports, since these classes may import from here
         from IPython.terminal.ipapp import TerminalIPythonApp
         apps = [TerminalIPythonApp]
-        try:
-            from IPython.qt.console.qtconsoleapp import IPythonQtConsoleApp
-        except Exception:
-            # this should be ImportError, but under weird circumstances
-            # this might be an AttributeError, or possibly others
-            # in any case, nothing should cause the profile creation to crash.
-            pass
-        else:
-            apps.append(IPythonQtConsoleApp)
-        try:
-            from IPython.html.notebookapp import NotebookApp
-        except ImportError:
-            pass
-        except Exception:
-            self.log.debug('Unexpected error when importing NotebookApp',
-                           exc_info=True
-            )
-        else:
-            apps.append(NotebookApp)
+        for app_path in (
+            'IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp',
+            'IPython.html.notebookapp.NotebookApp',
+            'IPython.nbconvert.nbconvertapp.NbConvertApp',
+        ):
+            app = self._import_app(app_path)
+            if app is not None:
+                apps.append(app)
         if self.parallel:
             from IPython.parallel.apps.ipcontrollerapp import IPControllerApp
             from IPython.parallel.apps.ipengineapp import IPEngineApp

IPython/core/pylabtools.py

@@ -214,7 +214,11 b' def find_gui_and_backend(gui=None, gui_select=None):'
         # select backend based on requested gui
         backend = backends[gui]
     else:
-        backend = matplotlib.rcParams['backend']
+        # We need to read the backend from the original data structure, *not*
+        # from mpl.rcParams, since a prior invocation of %matplotlib may have
+        # overwritten that.
+        # WARNING: this assumes matplotlib 1.1 or newer!!
+        backend = matplotlib.rcParamsOrig['backend']
         # In this case, we need to find what the appropriate gui selection call
         # should be for IPython, so we can activate inputhook accordingly
         gui = backend2gui.get(backend, None)

IPython/core/release.py

@@ -26,7 +26,7 b" _version_extra = 'dev'"
 # _version_extra = 'rc1'
 # _version_extra = ''  # Uncomment this for full releases
 
-codename = 'An Afternoon Hack'
+codename = 'Work in Progress'
 
 # Construct full version string from these.
 _ver = [_version_major, _version_minor, _version_patch]
@@ -141,11 +141,8 b' classifiers = ['
     'License :: OSI Approved :: BSD License',
     'Programming Language :: Python',
     'Programming Language :: Python :: 2',
-    'Programming Language :: Python :: 2.6',
     'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.2',
-    'Programming Language :: Python :: 3.3',
     'Topic :: System :: Distributed Computing',
     'Topic :: System :: Shells'
     ]

IPython/core/shellapp.py

@@ -36,11 +36,14 b' from IPython.utils.path import filefind'
 from IPython.utils.traitlets import (
     Unicode, Instance, List, Bool, CaselessStrEnum, Dict
 )
+from IPython.lib.inputhook import guis
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
+gui_keys = tuple(sorted([ key for key in guis if key is not None ]))
+
 backend_keys = sorted(pylabtools.backends.keys())
 backend_keys.insert(0, 'auto')
 
@@ -175,8 +178,8 b' class InteractiveShellApp(Configurable):'
     module_to_run = Unicode('', config=True,
         help="Run the module as a script."
     )
-    gui = CaselessStrEnum(('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx'), config=True,
-        help="Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx')."
+    gui = CaselessStrEnum(gui_keys, config=True,
+        help="Enable GUI event loop integration with any of {0}.".format(gui_keys)
     )
     matplotlib = CaselessStrEnum(backend_keys,
         config=True,
@@ -198,7 +201,7 b' class InteractiveShellApp(Configurable):'
     )
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     
-    user_ns = Dict(default_value=None)
+    user_ns = Instance(dict, args=None, allow_none=True)
     def _user_ns_changed(self, name, old, new):
         if self.shell is not None:
             self.shell.user_ns = new

IPython/core/tests/test_debugger.py

@@ -117,8 +117,7 b' def test_ipdb_magics():'
     ipdb> pinfo a
     Type:       ExampleClass
     String Form:ExampleClass()
-    Namespace:  Locals
-    File:       ...
+    Namespace:  Local...
     Docstring:  Docstring for ExampleClass.
     Constructor Docstring:Docstring for ExampleClass.__init__
     ipdb> continue

IPython/core/tests/test_handlers.py

@@ -45,28 +45,6 b' def run(tests):'
 
 
 def test_handlers():
-    # alias expansion
-
-    # We're using 'true' as our syscall of choice because it doesn't
-    # write anything to stdout.
-
-    # Turn off actual execution of aliases, because it's noisy
-    old_system_cmd = ip.system
-    ip.system = lambda cmd: None
-
-
-    ip.alias_manager.alias_table['an_alias'] = (0, 'true')
-    # These are useful for checking a particular recursive alias issue
-    ip.alias_manager.alias_table['top'] = (0, 'd:/cygwin/top')
-    ip.alias_manager.alias_table['d'] =   (0, 'true')
-    run([(i,py3compat.u_format(o)) for i,o in \
-        [("an_alias",    "get_ipython().system({u}'true ')"),     # alias
-         # Below: recursive aliases should expand whitespace-surrounded
-         # chars, *not* initial chars which happen to be aliases:
-         ("top",         "get_ipython().system({u}'d:/cygwin/top ')"),
-         ]])
-    ip.system = old_system_cmd
-
     call_idx = CallableIndexable()
     ip.user_ns['call_idx'] = call_idx
 

IPython/core/tests/test_imports.py

@@ -9,9 +9,6 b' def test_import_crashhandler():'
 def test_import_debugger():
     from IPython.core import debugger
 
-def test_import_fakemodule():
-    from IPython.core import fakemodule
-
 def test_import_excolors():
     from IPython.core import excolors
 

IPython/core/tests/test_inputtransformer.py

@@ -183,9 +183,18 b' syntax_ml = \\'
           ('... 456"""','456"""'),
           ],
          [('a="""','a="""'),
+          ('>>> 123','123'),
+          ('... 456"""','456"""'),
+          ],
+         [('a="""','a="""'),
           ('123','123'),
           ('... 456"""','... 456"""'),
           ],
+         [('....__class__','....__class__'),
+         ],
+         [('a=5', 'a=5'),
+          ('...', ''),
+         ],
          [('>>> def f(x):', 'def f(x):'),
           ('...', ''),
           ('...     return x', '    return x'),
@@ -205,6 +214,10 b' syntax_ml = \\'
           ('   ...: 456"""','456"""'),
           ],
          [('a="""','a="""'),
+          ('In [1]: 123','123'),
+          ('   ...: 456"""','456"""'),
+          ],
+         [('a="""','a="""'),
           ('123','123'),
           ('   ...: 456"""','   ...: 456"""'),
           ],
@@ -237,6 +250,12 b' syntax_ml = \\'
          ],
        ],
        
+       multiline_string =
+       [ [("'''foo?", None),
+          ("bar'''", "'''foo?\nbar'''"),
+         ],
+       ],
+       
        leading_indent =
        [ [('    print "hi"','print "hi"'),
           ],

IPython/core/tests/test_interactiveshell.py

@@ -106,17 +106,6 b' class InteractiveShellTestCase(unittest.TestCase):'
         ip.run_cell('a = """\n%exit\n"""')
         self.assertEqual(ip.user_ns['a'], '\n%exit\n')
     
-    def test_alias_crash(self):
-        """Errors in prefilter can't crash IPython"""
-        ip.run_cell('%alias parts echo first %s second %s')
-        # capture stderr:
-        save_err = io.stderr
-        io.stderr = StringIO()
-        ip.run_cell('parts 1')
-        err = io.stderr.getvalue()
-        io.stderr = save_err
-        self.assertEqual(err.split(':')[0], 'ERROR')
-    
     def test_trailing_newline(self):
         """test that running !(command) does not raise a SyntaxError"""
         ip.run_cell('!(true)\n', False)

IPython/core/tests/test_magic.py

@@ -48,16 +48,16 b' class DummyMagics(magic.Magics): pass'
 def test_rehashx():
     # clear up everything
     _ip = get_ipython()
-    _ip.alias_manager.alias_table.clear()
+    _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
-    nt.assert_true(len(_ip.alias_manager.alias_table) > 10)
-    for key, val in _ip.alias_manager.alias_table.iteritems():
+    nt.assert_true(len(_ip.alias_manager.aliases) > 10)
+    for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
-        nt.assert_not_in('.', key)
+        nt.assert_not_in('.', name)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
@@ -487,7 +487,21 b' def test_timeit_special_syntax():'
     # cell mode test
     _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
-    
+
+def test_timeit_return():
+    """
+    test wether timeit -o return object
+    """
+
+    res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
+    assert(res is not None)
+
+def test_timeit_quiet():
+    """
+    test quiet option of timeit magic
+    """
+    with tt.AssertNotPrints("loops"):
+        _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():

IPython/core/tests/test_magic_arguments.py

@@ -6,12 +6,11 b''
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
-from nose.tools import assert_equal, assert_true
+import argparse
+from nose.tools import assert_equal
 
-from IPython.external import argparse
 from IPython.core.magic_arguments import (argument, argument_group, kwds,
     magic_arguments, parse_argstring, real_name)
-from IPython.testing.decorators import parametric
 
 
 @magic_arguments()
@@ -74,48 +73,46 b' def foo(self, args):'
     return parse_argstring(foo, args)
 
 
-@parametric
 def test_magic_arguments():
-    # Ideally, these would be doctests, but I could not get it to work.
-    yield assert_equal(magic_foo1.__doc__, '%foo1 [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
-    yield assert_equal(getattr(magic_foo1, 'argcmd_name', None), None)
-    yield assert_equal(real_name(magic_foo1), 'foo1')
-    yield assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None))
-    yield assert_true(hasattr(magic_foo1, 'has_arguments'))
-
-    yield assert_equal(magic_foo2.__doc__, '%foo2\n\n A docstring.\n')
-    yield assert_equal(getattr(magic_foo2, 'argcmd_name', None), None)
-    yield assert_equal(real_name(magic_foo2), 'foo2')
-    yield assert_equal(magic_foo2(None, ''), argparse.Namespace())
-    yield assert_true(hasattr(magic_foo2, 'has_arguments'))
-
-    yield assert_equal(magic_foo3.__doc__, '%foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n\nGroup:\n  -b BAR, --bar BAR  a grouped argument\n\nSecond Group:\n  -z BAZ, --baz BAZ  another grouped argument\n')
-    yield assert_equal(getattr(magic_foo3, 'argcmd_name', None), None)
-    yield assert_equal(real_name(magic_foo3), 'foo3')
-    yield assert_equal(magic_foo3(None, ''),
+    assert_equal(magic_foo1.__doc__, '%foo1 [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(getattr(magic_foo1, 'argcmd_name', None), None)
+    assert_equal(real_name(magic_foo1), 'foo1')
+    assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None))
+    assert hasattr(magic_foo1, 'has_arguments')
+
+    assert_equal(magic_foo2.__doc__, '%foo2\n\n A docstring.\n')
+    assert_equal(getattr(magic_foo2, 'argcmd_name', None), None)
+    assert_equal(real_name(magic_foo2), 'foo2')
+    assert_equal(magic_foo2(None, ''), argparse.Namespace())
+    assert hasattr(magic_foo2, 'has_arguments')
+
+    assert_equal(magic_foo3.__doc__, '%foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n\nGroup:\n  -b BAR, --bar BAR  a grouped argument\n\nSecond Group:\n  -z BAZ, --baz BAZ  another grouped argument\n')
+    assert_equal(getattr(magic_foo3, 'argcmd_name', None), None)
+    assert_equal(real_name(magic_foo3), 'foo3')
+    assert_equal(magic_foo3(None, ''),
                        argparse.Namespace(bar=None, baz=None, foo=None))
-    yield assert_true(hasattr(magic_foo3, 'has_arguments'))
-
-    yield assert_equal(magic_foo4.__doc__, '%foo4 [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
-    yield assert_equal(getattr(magic_foo4, 'argcmd_name', None), None)
-    yield assert_equal(real_name(magic_foo4), 'foo4')
-    yield assert_equal(magic_foo4(None, ''), argparse.Namespace())
-    yield assert_true(hasattr(magic_foo4, 'has_arguments'))
-
-    yield assert_equal(magic_foo5.__doc__, '%frobnicate [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
-    yield assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate')
-    yield assert_equal(real_name(magic_foo5), 'frobnicate')
-    yield assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None))
-    yield assert_true(hasattr(magic_foo5, 'has_arguments'))
-
-    yield assert_equal(magic_magic_foo.__doc__, '%magic_foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
-    yield assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None)
-    yield assert_equal(real_name(magic_magic_foo), 'magic_foo')
-    yield assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None))
-    yield assert_true(hasattr(magic_magic_foo, 'has_arguments'))
-
-    yield assert_equal(foo.__doc__, '%foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
-    yield assert_equal(getattr(foo, 'argcmd_name', None), None)
-    yield assert_equal(real_name(foo), 'foo')
-    yield assert_equal(foo(None, ''), argparse.Namespace(foo=None))
-    yield assert_true(hasattr(foo, 'has_arguments'))
+    assert hasattr(magic_foo3, 'has_arguments')
+
+    assert_equal(magic_foo4.__doc__, '%foo4 [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(getattr(magic_foo4, 'argcmd_name', None), None)
+    assert_equal(real_name(magic_foo4), 'foo4')
+    assert_equal(magic_foo4(None, ''), argparse.Namespace())
+    assert hasattr(magic_foo4, 'has_arguments')
+
+    assert_equal(magic_foo5.__doc__, '%frobnicate [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate')
+    assert_equal(real_name(magic_foo5), 'frobnicate')
+    assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None))
+    assert hasattr(magic_foo5, 'has_arguments')
+
+    assert_equal(magic_magic_foo.__doc__, '%magic_foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None)
+    assert_equal(real_name(magic_magic_foo), 'magic_foo')
+    assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None))
+    assert hasattr(magic_magic_foo, 'has_arguments')
+
+    assert_equal(foo.__doc__, '%foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n  -f FOO, --foo FOO  an argument\n')
+    assert_equal(getattr(foo, 'argcmd_name', None), None)
+    assert_equal(real_name(foo), 'foo')
+    assert_equal(foo(None, ''), argparse.Namespace(foo=None))
+    assert hasattr(foo, 'has_arguments')

IPython/core/tests/test_prefilter.py

@@ -6,7 +6,6 b''
 import nose.tools as nt
 
 from IPython.core.prefilter import AutocallChecker
-from IPython.testing import decorators as dec
 from IPython.testing.globalipapp import get_ipython
 
 #-----------------------------------------------------------------------------
@@ -14,7 +13,6 b' from IPython.testing.globalipapp import get_ipython'
 #-----------------------------------------------------------------------------
 ip = get_ipython()
 
[e-mail]
 def test_prefilter():
     """Test user input conversions"""
 
@@ -23,19 +21,18 b' def test_prefilter():'
              ]
 
     for raw, correct in pairs:
-        yield nt.assert_equal(ip.prefilter(raw), correct)
+        nt.assert_equal(ip.prefilter(raw), correct)
 
 
[e-mail]
 def test_autocall_binops():
     """See https://github.com/ipython/ipython/issues/81"""
     ip.magic('autocall 2')
     f = lambda x: x
     ip.user_ns['f'] = f
     try:
-        yield nt.assert_equal(ip.prefilter('f 1'),'f(1)')
+        nt.assert_equal(ip.prefilter('f 1'),'f(1)')
         for t in ['f +1', 'f -1']:
-            yield nt.assert_equal(ip.prefilter(t), t)
+            nt.assert_equal(ip.prefilter(t), t)
 
         # Run tests again with a more permissive exclude_regexp, which will
         # allow transformation of binary operations ('f -1' -> 'f(-1)').
@@ -47,8 +44,8 b' def test_autocall_binops():'
             ac.exclude_regexp = r'^[,&^\|\*/]|^is |^not |^in |^and |^or '
             pm.sort_checkers()
 
-            yield nt.assert_equal(ip.prefilter('f -1'), 'f(-1)')
-            yield nt.assert_equal(ip.prefilter('f +1'), 'f(+1)')
+            nt.assert_equal(ip.prefilter('f -1'), 'f(-1)')
+            nt.assert_equal(ip.prefilter('f +1'), 'f(+1)')
         finally:
             pm.unregister_checker(ac)
     finally:
@@ -56,7 +53,6 b' def test_autocall_binops():'
         del ip.user_ns['f']
 
 
[e-mail]
 def test_issue_114():
     """Check that multiline string literals don't expand as magic
     see http://github.com/ipython/ipython/issues/114"""
@@ -70,7 +66,7 b' def test_issue_114():'
     try:
         for mgk in ip.magics_manager.lsmagic()['line']:
             raw = template % mgk
-            yield nt.assert_equal(ip.prefilter(raw), raw)
+            nt.assert_equal(ip.prefilter(raw), raw)
     finally:
         ip.prefilter_manager.multi_line_specials = msp
 

IPython/core/tests/test_pylabtools.py

@@ -24,7 +24,6 b' import numpy as np'
 
 # Our own imports
 from IPython.core.interactiveshell import InteractiveShell
-from IPython.testing import decorators as dec
 from .. import pylabtools as pt
 
 #-----------------------------------------------------------------------------
@@ -39,11 +38,10 b' from .. import pylabtools as pt'
 # Classes and functions
 #-----------------------------------------------------------------------------
 
[e-mail]
 def test_figure_to_svg():
     # simple empty-figure test
     fig = plt.figure()
-    yield nt.assert_equal(pt.print_figure(fig, 'svg'), None)
+    nt.assert_equal(pt.print_figure(fig, 'svg'), None)
 
     plt.close('all')
 
@@ -53,7 +51,7 b' def test_figure_to_svg():'
     ax.plot([1,2,3])
     plt.draw()
     svg = pt.print_figure(fig, 'svg')[:100].lower()
-    yield nt.assert_true('doctype svg' in svg)
+    nt.assert_in(b'doctype svg', svg)
 
 
 def test_import_pylab():

IPython/core/tests/test_run.py

@@ -15,6 +15,7 b' from __future__ import absolute_import'
 
 import functools
 import os
+from os.path import join as pjoin
 import random
 import sys
 import tempfile
@@ -359,6 +360,17 b' tclass.py: deleting object: C-third'
         self.mktmp(src)
         _ip.magic('run -t -N 1 %s' % self.fname)
         _ip.magic('run -t -N 10 %s' % self.fname)
+    
+    def test_ignore_sys_exit(self):
+        """Test the -e option to ignore sys.exit()"""
+        src = "import sys; sys.exit(1)"
+        self.mktmp(src)
+        with tt.AssertPrints('SystemExit'):
+            _ip.magic('run %s' % self.fname)
+        
+        with tt.AssertNotPrints('SystemExit'):
+            _ip.magic('run -e %s' % self.fname)
+        
 
 
 class TestMagicRunWithPackage(unittest.TestCase):
@@ -398,6 +410,7 b' class TestMagicRunWithPackage(unittest.TestCase):'
         self.tempdir.cleanup()
 
     def check_run_submodule(self, submodule, opts=''):
+        _ip.user_ns.pop('x', None)
         _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts))
         self.assertEqual(_ip.user_ns['x'], self.value,
                          'Variable `x` is not loaded from module `{0}`.'
@@ -430,3 +443,16 b' class TestMagicRunWithPackage(unittest.TestCase):'
     @with_fake_debugger
     def test_debug_run_submodule_with_relative_import(self):
         self.check_run_submodule('relative', '-d')
+
+def test_run__name__():
+    with TemporaryDirectory() as td:
+        path = pjoin(td, 'foo.py')
+        with open(path, 'w') as f:
+            f.write("q = __name__")
+        
+        _ip.user_ns.pop('q', None)
+        _ip.magic('run {}'.format(path))
+        nt.assert_equal(_ip.user_ns.pop('q'), '__main__')
+        
+        _ip.magic('run -n {}'.format(path))
+        nt.assert_equal(_ip.user_ns.pop('q'), 'foo')

IPython/core/tests/test_ultratb.py

@@ -108,8 +108,33 b' class IndentationErrorTest(unittest.TestCase):'
                 with tt.AssertPrints("zoon()", suppress=False):
                     ip.magic('run %s' % fname)
 
+se_file_1 = """1
+2
+7/
+"""
+
+se_file_2 = """7/
+"""
+
 class SyntaxErrorTest(unittest.TestCase):
     def test_syntaxerror_without_lineno(self):
         with tt.AssertNotPrints("TypeError"):
             with tt.AssertPrints("line unknown"):
                 ip.run_cell("raise SyntaxError()")
+
+    def test_changing_py_file(self):
+        with TemporaryDirectory() as td:
+            fname = os.path.join(td, "foo.py")
+            with open(fname, 'w') as f:
+                f.write(se_file_1)
+
+            with tt.AssertPrints(["7/", "SyntaxError"]):
+                ip.magic("run " + fname)
+
+            # Modify the file
+            with open(fname, 'w') as f:
+                f.write(se_file_2)
+
+            # The SyntaxError should point to the correct line
+            with tt.AssertPrints(["7/", "SyntaxError"]):
+                ip.magic("run " + fname)

IPython/core/ultratb.py

@@ -8,7 +8,8 b' ColorTB class is a solution to that problem. It colors the different parts of a'
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
-Installation instructions for ColorTB:
+Installation instructions for ColorTB::
+
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
@@ -21,7 +22,7 b' but kind of neat, and maybe useful for long-running programs that you believe'
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
-Note:
+.. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
@@ -36,25 +37,28 b' Note:'
   Verbose).
 
 
-Installation instructions for ColorTB:
+Installation instructions for ColorTB::
+
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
-* Color schemes
+Color schemes
+-------------
+
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
-  escapes are just dummy blank strings).
+    escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
-  or very dark background).
+    or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
-  in light background terminals.
+    in light background terminals.
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
@@ -359,8 +363,8 b' class TBTools(object):'
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
-        to io.stdout.  This ensures compatibility with most tools, including
-        Windows (where plain stdout doesn't recognize ANSI escapes).
+          to io.stdout.  This ensures compatibility with most tools, including
+          Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
@@ -974,9 +978,9 b' class VerboseTB(TBTools):'
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
-          flag and does not actually invoke the debugger if the flag is false.
-          The 'force' option forces the debugger to activate even if the flag
-          is false.
+            flag and does not actually invoke the debugger if the flag is false.
+            The 'force' option forces the debugger to activate even if the flag
+            is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
@@ -1193,6 +1197,20 b' class SyntaxTB(ListTB):'
         self.last_syntax_error = value
         ListTB.__call__(self,etype,value,elist)
 
+    def structured_traceback(self, etype, value, elist, tb_offset=None,
+                             context=5):
+        # If the source file has been edited, the line in the syntax error can
+        # be wrong (retrieved from an outdated cache). This replaces it with
+        # the current value.
+        if isinstance(value.filename, py3compat.string_types) \
+                and isinstance(value.lineno, int):
+            linecache.checkcache(value.filename)
+            newtext = ulinecache.getline(value.filename, value.lineno)
+            if newtext:
+                value.text = newtext
+        return super(SyntaxTB, self).structured_traceback(etype, value, elist,
+                             tb_offset=tb_offset, context=context)
+
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error

IPython/extensions/cythonmagic.py

@@ -4,9 +4,18 b''
 Cython related magics
 =====================
 
+Magic command interface for interactive work with Cython
+
+.. note::
+
+  The ``Cython`` package needs to be installed separately. It
+  can be obtained using ``easy_install`` or ``pip``.
+
 Usage
 =====
 
+To enable the magics below, execute ``%load_ext cythonmagic``.
+
 ``%%cython``
 
 {CYTHON_DOC}

IPython/extensions/octavemagic.py

@@ -11,9 +11,13 b' Magics for interacting with Octave via oct2py.'
   The ``oct2py`` module needs to be installed separately and
   can be obtained using ``easy_install`` or ``pip``.
 
+  You will also need a working copy of GNU Octave.
+
 Usage
 =====
 
+To enable the magics below, execute ``%load_ext octavemagic``.
+
 ``%octave``
 
 {OCTAVE_DOC}

IPython/extensions/rmagic.py

@@ -6,9 +6,18 b' Rmagic'
 
 Magic command interface for interactive work with R via rpy2
 
+.. note::
+
+  The ``rpy2`` package needs to be installed separately. It
+  can be obtained using ``easy_install`` or ``pip``.
+
+  You will also need a working copy of R.
+
 Usage
 =====
 
+To enable the magics below, execute ``%load_ext rmagic``.
+
 ``%R``
 
 {R_DOC}
@@ -425,12 +434,12 b' class RMagics(Magics):'
         is printed if it would printed be when evaluating the same code 
         within a standard R REPL.
         
-        Nothing is returned to python by default in cell mode.
+        Nothing is returned to python by default in cell mode::
 
             In [10]: %%R
                ....: Y = c(2,4,3,9)
                ....: summary(lm(Y~X))
-       
+
             Call:
             lm(formula = Y ~ X)
 
@@ -447,9 +456,9 b' class RMagics(Magics):'
             Multiple R-squared: 0.6993,Adjusted R-squared: 0.549
             F-statistic: 4.651 on 1 and 2 DF,  p-value: 0.1638
 
-        In the notebook, plots are published as the output of the cell.
+        In the notebook, plots are published as the output of the cell::
 
-        %R plot(X, Y)
+            %R plot(X, Y)
 
         will create a scatter plot of X bs Y.
 
@@ -475,19 +484,19 b' class RMagics(Magics):'
         * If the cell is not None, the magic returns None.
 
         * If the cell evaluates as False, the resulting value is returned
-        unless the final line prints something to the console, in
-        which case None is returned.
+          unless the final line prints something to the console, in
+          which case None is returned.
 
         * If the final line results in a NULL value when evaluated
-        by rpy2, then None is returned.
+          by rpy2, then None is returned.
 
         * No attempt is made to convert the final value to a structured array.
-        Use the --dataframe flag or %Rget to push / return a structured array.
+          Use the --dataframe flag or %Rget to push / return a structured array.
 
         * If the -n flag is present, there is no return value.
 
         * A trailing ';' will also result in no return value as the last
-        value in the line is an empty string.
+          value in the line is an empty string.
 
         The --dataframe argument will attempt to return structured arrays. 
         This is useful for dataframes with

IPython/extensions/storemagic.py

@@ -25,10 +25,11 b' To automatically restore stored variables at startup, add this to your'
 import inspect, os, sys, textwrap
 
 # Our own
+from IPython.config.configurable import Configurable
 from IPython.core.error import UsageError
-from IPython.core.fakemodule import FakeModule
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.testing.skipdoctest import skip_doctest
+from IPython.utils.traitlets import Bool
 
 #-----------------------------------------------------------------------------
 # Functions and classes
@@ -68,10 +69,23 b' def restore_data(ip):'
 
 
 @magics_class
-class StoreMagics(Magics):
+class StoreMagics(Magics, Configurable):
     """Lightweight persistence for python variables.
 
     Provides the %store magic."""
+    
+    autorestore = Bool(False, config=True, help=
+        """If True, any %store-d variables will be automatically restored
+        when IPython starts.
+        """
+    )
+    
+    def __init__(self, shell):
+        Configurable.__init__(self, config=shell.config)
+        Magics.__init__(self, shell=shell)
+        self.shell.configurables.append(self)
+        if self.autorestore:
+            restore_data(self.shell)
 
     @skip_doctest
     @line_magic
@@ -196,20 +210,21 b' class StoreMagics(Magics):'
                 obj = ip.user_ns[args[0]]
             except KeyError:
                 # it might be an alias
-                # This needs to be refactored to use the new AliasManager stuff.
-                if args[0] in ip.alias_manager:
-                    name = args[0]
-                    nargs, cmd = ip.alias_manager.alias_table[ name ]
-                    staliases = db.get('stored_aliases',{})
-                    staliases[ name ] = cmd
-                    db['stored_aliases'] = staliases
-                    print "Alias stored: %s (%s)" % (name, cmd)
-                    return
-                else:
-                    raise UsageError("Unknown variable '%s'" % args[0])
+                name = args[0]
+                try:
+                    cmd = ip.alias_manager.retrieve_alias(name)
+                except ValueError:
+                    raise UsageError("Unknown variable '%s'" % name)
+                
+                staliases = db.get('stored_aliases',{})
+                staliases[name] = cmd
+                db['stored_aliases'] = staliases
+                print "Alias stored: %s (%s)" % (name, cmd)
+                return
 
             else:
-                if isinstance(inspect.getmodule(obj), FakeModule):
+                modname = getattr(inspect.getmodule(obj), '__name__', '')
+                if modname == '__main__':
                     print textwrap.dedent("""\
                     Warning:%s is %s
                     Proper storage of interactively declared classes (or instances
@@ -225,3 +240,4 b' class StoreMagics(Magics):'
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
     ip.register_magics(StoreMagics)
+    

IPython/extensions/tests/test_storemagic.py

@@ -1,5 +1,6 b''
 import tempfile, os
 
+from IPython.config.loader import Config
 import nose.tools as nt
 
 ip = get_ipython()
@@ -26,7 +27,24 b' def test_store_restore():'
     # Check restoring
     ip.magic('store -r')
     nt.assert_equal(ip.user_ns['foo'], 78)
-    nt.assert_in('bar', ip.alias_manager.alias_table)
+    assert ip.alias_manager.is_alias('bar')
     nt.assert_in(os.path.realpath(tmpd), ip.user_ns['_dh'])
     
     os.rmdir(tmpd)
+
+def test_autorestore():
+    ip.user_ns['foo'] = 95
+    ip.magic('store foo')
+    del ip.user_ns['foo']
+    c = Config()
+    c.StoreMagics.autorestore = False
+    orig_config = ip.config
+    try:
+        ip.config = c
+        ip.extension_manager.reload_extension('storemagic')
+        nt.assert_not_in('foo', ip.user_ns)
+        c.StoreMagics.autorestore = True
+        ip.extension_manager.reload_extension('storemagic')
+        nt.assert_equal(ip.user_ns['foo'], 95)
+    finally:
+        ip.config = orig_config

IPython/external/mathjax.py

@@ -19,6 +19,10 b' From the command line:'
 
     $ python -m IPython.external.mathjax
 
+To a specific profile:
+
+    $ python -m IPython.external.mathjax --profile=research
+
 To install MathJax from a file you have already downloaded:
 
     $ python -m IPython.external.mathjax mathjax-xxx.tar.gz
@@ -46,6 +50,7 b' To find the directory where IPython would like MathJax installed:'
 # Imports
 #-----------------------------------------------------------------------------
 
+import argparse
 import os
 import shutil
 import sys
@@ -55,7 +60,6 b' import zipfile'
 
 
 from IPython.utils.path import locate_profile
-from IPython.external import argparse
 #-----------------------------------------------------------------------------
 #
 #-----------------------------------------------------------------------------
@@ -200,20 +204,26 b' def main() :'
             )
 
     parser.add_argument(
+            '-p',
+            '--profile',
+            default='default',
+            help='profile to install MathJax to (default is default)')
+
+    parser.add_argument(
             '-i',
             '--install-dir',
-            default=default_dest,
-            help='installation directory (by default : %s)' % (default_dest))
+            help='custom installation directory')
+
     parser.add_argument(
             '-d',
             '--dest',
             action='store_true',
-            help='print where is current mathjax would be installed and exit')
+            help='print where current mathjax would be installed and exit')
     parser.add_argument(
             '-r',
             '--replace',
             action='store_true',
-            help='Wether to replace current mathjax if already exist')
+            help='Whether to replace current mathjax if it already exists')
     parser.add_argument(
             '-t',
             '--test',
@@ -225,7 +235,13 b' def main() :'
 
     pargs = parser.parse_args()
 
-    dest = pargs.install_dir
+    if pargs.install_dir:
+        # Explicit install_dir overrides profile
+        dest = pargs.install_dir
+    else:
+        profile = pargs.profile
+        dest = os.path.join(locate_profile(profile), 'static', 'mathjax')
+
     if pargs.dest :
         print dest
         return
@@ -259,7 +275,7 b' def main() :'
 if __name__ == '__main__' :
     sys.exit(main())
 
-__all__ = ['install_mathjax', 'main', 'dest']
+__all__ = ['install_mathjax', 'main', 'default_dest']
 
 """
 Test notes:

IPython/html/notebookapp.py

@@ -78,7 +78,7 b' from IPython.kernel.zmq.kernelapp import ('
     kernel_aliases,
 )
 from IPython.utils.importstring import import_item
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils import submodule
 from IPython.utils.traitlets import (
     Dict, Unicode, Integer, List, Bool, Bytes,
@@ -253,7 +253,7 b' aliases.update({'
 aliases.pop('f', None)
 
 notebook_aliases = [u'port', u'port-retries', u'ip', u'keyfile', u'certfile',
-                    u'notebook-dir']
+                    u'notebook-dir', u'profile', u'profile-dir']
 
 #-----------------------------------------------------------------------------
 # NotebookApp
@@ -293,9 +293,11 b' class NotebookApp(BaseIPythonApplication):'
 
     # Network related information.
 
-    ip = Unicode(LOCALHOST, config=True,
+    ip = Unicode(config=True,
         help="The IP address the notebook server will listen on."
     )
+    def _ip_default(self):
+        return localhost()
 
     def _ip_changed(self, name, old, new):
         if new == u'*': self.ip = u''
@@ -471,17 +473,10 b' class NotebookApp(BaseIPythonApplication):'
         help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers"
               "sent by the upstream reverse proxy. Neccesary if the proxy handles SSL")
     )
-
+    
     def parse_command_line(self, argv=None):
         super(NotebookApp, self).parse_command_line(argv)
-        if argv is None:
-            argv = sys.argv[1:]
-
-        # Scrub frontend-specific flags
-        self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
-        # Kernel should inherit default config file from frontend
-        self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name)
-
+        
         if self.extra_args:
             f = os.path.abspath(self.extra_args[0])
             if os.path.isdir(f):
@@ -491,6 +486,15 b' class NotebookApp(BaseIPythonApplication):'
                 nbdir = os.path.dirname(f)
             self.config.NotebookManager.notebook_dir = nbdir
 
+    def init_kernel_argv(self):
+        """construct the kernel arguments"""
+        # Scrub frontend-specific flags
+        self.kernel_argv = swallow_argv(self.argv, notebook_aliases, notebook_flags)
+        # Kernel should inherit default config file from frontend
+        self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name)
+        # Kernel should get *absolute* path to profile directory
+        self.kernel_argv.extend(["--profile-dir", self.profile_dir.location])
+
     def init_configurables(self):
         # force Session default to be secure
         default_secure(self.config)
@@ -657,6 +661,7 b' class NotebookApp(BaseIPythonApplication):'
     def initialize(self, argv=None):
         self.init_logging()
         super(NotebookApp, self).initialize(argv)
+        self.init_kernel_argv()
         self.init_configurables()
         self.init_components()
         self.init_webapp()
@@ -691,7 +696,7 b' class NotebookApp(BaseIPythonApplication):'
         info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).")
 
         if self.open_browser or self.file_to_run:
-            ip = self.ip or LOCALHOST
+            ip = self.ip or localhost()
             try:
                 browser = webbrowser.get(self.browser or None)
             except webbrowser.Error as e:

IPython/html/static/base/less/style.less

@@ -1,4 +1,4 b''
 @import "../base/less/variables.less";
 @import "../base/less/mixins.less";
 @import "../base/less/flexbox.less";
-@import "../base/less/page.less";
+

IPython/html/static/custom/custom.js

@@ -17,12 +17,14 b''
  * Create a custom button in toolbar that execute `%qtconsole` in kernel
  * and hence open a qtconsole attached to the same kernel as the current notebook
  *
- *    $([IPython.events]).on('notebook_loaded.Notebook', function(){
+ *    $([IPython.events]).on('app_initialized.NotebookApp', function(){
  *        IPython.toolbar.add_buttons_group([
  *            {
  *                 'label'   : 'run qtconsole',
- *                 'icon'    : 'ui-icon-calculator', // select your icon from http://jqueryui.com/themeroller/
- *                 'callback': function(){IPython.notebook.kernel.execute('%qtconsole')}
+ *                 'icon'    : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons
+ *                 'callback': function () {
+ *                     IPython.notebook.kernel.execute('%qtconsole')
+ *                 }
  *            }
  *            // add more button here if needed.
  *            ]);

IPython/html/static/notebook/js/cell.js

@@ -207,7 +207,7 b' var IPython = (function (IPython) {'
 
 
     /**
-     * can the cell be splitted in 2 cells.
+     * can the cell be split into two cells
      * @method is_splittable
      **/
     Cell.prototype.is_splittable = function () {
@@ -216,6 +216,15 b' var IPython = (function (IPython) {'
 
 
     /**
+     * can the cell be merged with other cells
+     * @method is_mergeable
+     **/
+    Cell.prototype.is_mergeable = function () {
+        return true;
+    };
+
+
+    /**
      * @return {String} - the text before the cursor
      * @method get_pre_cursor
      **/
@@ -323,8 +332,13 b' var IPython = (function (IPython) {'
                 }
             }
         }
-        // fallback on default (python)
-        var default_mode = this.default_mode || 'text/plain';
+        // fallback on default
+        var default_mode
+        try {
+            default_mode = this._options.cm_config.mode;
+        } catch(e) {
+            default_mode = 'text/plain';
+        }
         this.code_mirror.setOption('mode', default_mode);
     };
 

IPython/html/static/notebook/js/codecell.js

@@ -65,8 +65,8 b' var IPython = (function (IPython) {'
         this.code_mirror = null;
         this.input_prompt_number = null;
         this.collapsed = false;
-        this.default_mode = 'ipython';
         this.cell_type = "code";
+        this.last_msg_id = null;
 
 
         var cm_overwrite_options  = {
@@ -167,7 +167,7 b' var IPython = (function (IPython) {'
             // triger on keypress (!) otherwise inconsistent event.which depending on plateform
             // browser and keyboard layout !
             // Pressing '(' , request tooltip, don't forget to reappend it
-            // The second argument says to hide the tooltip if the docstring 
+            // The second argument says to hide the tooltip if the docstring
             // is actually empty
             IPython.tooltip.pending(that, true);
         } else if (event.which === key.UPARROW && event.type === 'keydown') {
@@ -180,8 +180,7 b' var IPython = (function (IPython) {'
                 return true;
             };
         } else if (event.which === key.ESC) {
-            IPython.tooltip.remove_and_cancel_tooltip(true);
-            return true;
+            return IPython.tooltip.remove_and_cancel_tooltip(true);
         } else if (event.which === key.DOWNARROW && event.type === 'keydown') {
             // If we are not at the bottom, let CM handle the down arrow and
             // prevent the global keydown handler from handling it.
@@ -242,9 +241,12 b' var IPython = (function (IPython) {'
      * @method execute
      */
     CodeCell.prototype.execute = function () {
-        this.output_area.clear_output(true, true, true);
+        this.output_area.clear_output();
         this.set_input_prompt('*');
         this.element.addClass("running");
+        if (this.last_msg_id) {
+            this.kernel.clear_callbacks_for_msg(this.last_msg_id);
+        }
         var callbacks = {
             'execute_reply': $.proxy(this._handle_execute_reply, this),
             'output': $.proxy(this.output_area.handle_output, this.output_area),
@@ -252,7 +254,7 b' var IPython = (function (IPython) {'
             'set_next_input': $.proxy(this._handle_set_next_input, this),
             'input_request': $.proxy(this._handle_input_request, this)
         };
-        var msg_id = this.kernel.execute(this.get_text(), callbacks, {silent: false, store_history: true});
+        this.last_msg_id = this.kernel.execute(this.get_text(), callbacks, {silent: false, store_history: true});
     };
 
     /**
@@ -273,7 +275,7 b' var IPython = (function (IPython) {'
         var data = {'cell': this, 'text': text}
         $([IPython.events]).trigger('set_next_input.Notebook', data);
     }
-    
+
     /**
      * @method _handle_input_request
      * @private
@@ -388,8 +390,8 b' var IPython = (function (IPython) {'
     };
 
 
-    CodeCell.prototype.clear_output = function (stdout, stderr, other) {
-        this.output_area.clear_output(stdout, stderr, other);
+    CodeCell.prototype.clear_output = function (wait) {
+        this.output_area.clear_output(wait);
     };
 
 

IPython/html/static/notebook/js/config.js

@@ -50,12 +50,13 b' var IPython = (function (IPython) {'
          *     cell_magic_highlight['javascript'] = {'reg':[/^var/]}
          */
         cell_magic_highlight : {
-              'magic_javascript':{'reg':[/^%%javascript/]}
-             ,'magic_perl'      :{'reg':[/^%%perl/]}
-             ,'magic_ruby'      :{'reg':[/^%%ruby/]}
-             ,'magic_python'    :{'reg':[/^%%python3?/]}
-             ,'magic_shell'      :{'reg':[/^%%bash/]}
-             ,'magic_r'         :{'reg':[/^%%R/]}
+              'magic_javascript'    :{'reg':[/^%%javascript/]}
+             ,'magic_perl'          :{'reg':[/^%%perl/]}
+             ,'magic_ruby'          :{'reg':[/^%%ruby/]}
+             ,'magic_python'        :{'reg':[/^%%python3?/]}
+             ,'magic_shell'         :{'reg':[/^%%bash/]}
+             ,'magic_r'             :{'reg':[/^%%R/]}
+             ,'magic_text/x-cython' :{'reg':[/^%%cython/]}
             },
 
         /**

IPython/html/static/notebook/js/main.js

@@ -99,6 +99,10 b' function (marked) {'
             tables: true,
             langPrefix: "language-",
             highlight: function(code, lang) {
+                if (!lang) {
+                    // no language, no highlight
+                    return code;
+                }
                 var highlighted;
                 try {
                     highlighted = hljs.highlight(lang, code, false);

IPython/html/static/notebook/js/notebook.js

@@ -1175,8 +1175,14 b' var IPython = (function (IPython) {'
     Notebook.prototype.merge_cell_above = function () {
         var index = this.get_selected_index();
         var cell = this.get_cell(index);
+        if (!cell.is_mergeable()) {
+            return;
+        }
         if (index > 0) {
             var upper_cell = this.get_cell(index-1);
+            if (!upper_cell.is_mergeable()) {
+                return;
+            }
             var upper_text = upper_cell.get_text();
             var text = cell.get_text();
             if (cell instanceof IPython.CodeCell) {
@@ -1199,8 +1205,14 b' var IPython = (function (IPython) {'
     Notebook.prototype.merge_cell_below = function () {
         var index = this.get_selected_index();
         var cell = this.get_cell(index);
+        if (!cell.is_mergeable()) {
+            return;
+        }
         if (index < this.ncells()-1) {
             var lower_cell = this.get_cell(index+1);
+            if (!lower_cell.is_mergeable()) {
+                return;
+            }
             var lower_text = lower_cell.get_text();
             var text = cell.get_text();
             if (cell instanceof IPython.CodeCell) {
@@ -1327,7 +1339,7 b' var IPython = (function (IPython) {'
         var cells = this.get_cells();
         for (var i=0; i<ncells; i++) {
             if (cells[i] instanceof IPython.CodeCell) {
-                cells[i].clear_output(true,true,true);
+                cells[i].clear_output();
                 // Make all In[] prompts blank, as well
                 // TODO: make this configurable (via checkbox?)
                 cells[i].set_input_prompt();

IPython/html/static/notebook/js/outputarea.js

@@ -31,7 +31,7 b' var IPython = (function (IPython) {'
         this.outputs = [];
         this.collapsed = false;
         this.scrolled = false;
-        this.clear_out_timeout = null;
+        this.clear_queued = null;
         if (prompt_area === undefined) {
             this.prompt_area = true;
         } else {
@@ -289,7 +289,12 b' var IPython = (function (IPython) {'
     OutputArea.prototype.append_output = function (json, dynamic) {
         // If dynamic is true, javascript output will be eval'd.
         this.expand();
-        this.flush_clear_timeout();
+
+        // Clear the output if clear is queued.
+        if (this.clear_queued) {
+            this.clear_output(false);
+        }
+
         if (json.output_type === 'pyout') {
             this.append_pyout(json, dynamic);
         } else if (json.output_type === 'pyerr') {
@@ -300,6 +305,7 b' var IPython = (function (IPython) {'
             this.append_stream(json);
         }
         this.outputs.push(json);
+        this.element.height('auto');
         var that = this;
         setTimeout(function(){that.element.trigger('resize');}, 100);
     };
@@ -312,6 +318,33 b' var IPython = (function (IPython) {'
         }
         return oa;
     };
+    
+    OutputArea.prototype._append_javascript_error = function (err, container) {
+        // display a message when a javascript error occurs in display output
+        var msg = "Javascript error adding output!"
+        console.log(msg, err);
+        if ( container === undefined ) return;
+        container.append(
+            $('<div/>').html(msg + "<br/>" +
+                err.toString() +
+                '<br/>See your browser Javascript console for more details.'
+            ).addClass('js-error')
+        );
+        container.show();
+    };
+    
+    OutputArea.prototype._safe_append = function (toinsert) {
+        // safely append an item to the document
+        // this is an object created by user code,
+        // and may have errors, which should not be raised
+        // under any circumstances.
+        try {
+            this.element.append(toinsert);
+        } catch(err) {
+            console.log(err);
+            this._append_javascript_error(err, this.element);
+        }
+    };
 
 
     OutputArea.prototype.append_pyout = function (json, dynamic) {
@@ -321,19 +354,7 b' var IPython = (function (IPython) {'
             toinsert.find('div.prompt').addClass('output_prompt').html('Out[' + n + ']:');
         }
         this.append_mime_type(json, toinsert, dynamic);
-        try {
-            this.element.append(toinsert);
-        } catch(err) {
-            console.log("Error attaching output!");
-            console.log(err);
-            this.element.show();
-            toinsert.html($('<div/>')
-                .html("Javascript error adding output!<br/>" +
-                    err.toString() +
-                    '<br/>See your browser Javascript console for more details.')
-                .addClass('js-error')
-            );
-        }
+        this._safe_append(toinsert);
         // If we just output latex, typeset it.
         if ((json.latex !== undefined) || (json.html !== undefined)) {
             this.typeset();
@@ -352,7 +373,7 b' var IPython = (function (IPython) {'
             s = s + '\n';
             var toinsert = this.create_output_area();
             this.append_text(s, {}, toinsert);
-            this.element.append(toinsert);
+            this._safe_append(toinsert);
         }
     };
 
@@ -389,14 +410,14 b' var IPython = (function (IPython) {'
         // If we got here, attach a new div
         var toinsert = this.create_output_area();
         this.append_text(text, {}, toinsert, "output_stream "+subclass);
-        this.element.append(toinsert);
+        this._safe_append(toinsert);
     };
 
 
     OutputArea.prototype.append_display_data = function (json, dynamic) {
         var toinsert = this.create_output_area();
         this.append_mime_type(json, toinsert, dynamic);
-        this.element.append(toinsert);
+        this._safe_append(toinsert);
         // If we just output latex, typeset it.
         if ( (json.latex !== undefined) || (json.html !== undefined) ) {
             this.typeset();
@@ -444,15 +465,7 b' var IPython = (function (IPython) {'
         try {
             eval(js);
         } catch(err) {
-            console.log('Error in Javascript!');
-            console.log(err);
-            container.show();
-            element.append($('<div/>')
-                .html("Error in Javascript !<br/>"+
-                    err.toString()+
-                    '<br/>See your browser Javascript console for more details.')
-                .addClass('js-error')
-                );
+            this._append_javascript_error(err, container);
         }
     };
 
@@ -546,7 +559,6 b' var IPython = (function (IPython) {'
     OutputArea.prototype.append_raw_input = function (content) {
         var that = this;
         this.expand();
-        this.flush_clear_timeout();
         var area = this.create_output_area();
         
         // disable any other raw_inputs, if they are left around
@@ -599,81 +611,35 b' var IPython = (function (IPython) {'
 
 
     OutputArea.prototype.handle_clear_output = function (content) {
-        this.clear_output(content.stdout, content.stderr, content.other);
+        this.clear_output(content.wait);
     };
 
 
-    OutputArea.prototype.clear_output = function (stdout, stderr, other) {
-        var that = this;
-        if (this.clear_out_timeout != null){
-            // fire previous pending clear *immediately*
-            clearTimeout(this.clear_out_timeout);
-            this.clear_out_timeout = null;
-            this.clear_output_callback(this._clear_stdout, this._clear_stderr, this._clear_other);
-        }
-        // store flags for flushing the timeout
-        this._clear_stdout = stdout;
-        this._clear_stderr = stderr;
-        this._clear_other = other;
-        this.clear_out_timeout = setTimeout(function() {
-            // really clear timeout only after a short delay
-            // this reduces flicker in 'clear_output; print' cases
-            that.clear_out_timeout = null;
-            that._clear_stdout = that._clear_stderr = that._clear_other = null;
-            that.clear_output_callback(stdout, stderr, other);
-        }, 500
-        );
-    };
+    OutputArea.prototype.clear_output = function(wait) {
+        if (wait) {
 
+            // If a clear is queued, clear before adding another to the queue.
+            if (this.clear_queued) {
+                this.clear_output(false);
+            };
 
-    OutputArea.prototype.clear_output_callback = function (stdout, stderr, other) {
-        var output_div = this.element;
+            this.clear_queued = true;
+        } else {
 
-        if (stdout && stderr && other){
+            // Fix the output div's height if the clear_output is waiting for
+            // new output (it is being used in an animation).
+            if (this.clear_queued) {
+                var height = this.element.height();
+                this.element.height(height);
+                this.clear_queued = false;
+            }
+            
             // clear all, no need for logic
-            output_div.html("");
+            this.element.html("");
             this.outputs = [];
             this.unscroll_area();
             return;
-        }
-        // remove html output
-        // each output_subarea that has an identifying class is in an output_area
-        // which is the element to be removed.
-        if (stdout) {
-            output_div.find("div.output_stdout").parent().remove();
-        }
-        if (stderr) {
-            output_div.find("div.output_stderr").parent().remove();
-        }
-        if (other) {
-            output_div.find("div.output_subarea").not("div.output_stderr").not("div.output_stdout").parent().remove();
-        }
-        this.unscroll_area();
-        
-        // remove cleared outputs from JSON list:
-        for (var i = this.outputs.length - 1; i >= 0; i--) {
-            var out = this.outputs[i];
-            var output_type = out.output_type;
-            if (output_type == "display_data" && other) {
-                this.outputs.splice(i,1);
-            } else if (output_type == "stream") {
-                if (stdout && out.stream == "stdout") {
-                    this.outputs.splice(i,1);
-                } else if (stderr && out.stream == "stderr") {
-                    this.outputs.splice(i,1);
-                }
-            }
-        }
-    };
-
-
-    OutputArea.prototype.flush_clear_timeout = function() {
-        var output_div = this.element;
-        if (this.clear_out_timeout){
-            clearTimeout(this.clear_out_timeout);
-            this.clear_out_timeout = null;
-            this.clear_output_callback(this._clear_stdout, this._clear_stderr, this._clear_other);
-        }
+        };
     };
 
 

IPython/html/static/notebook/js/savewidget.js

@@ -52,7 +52,7 b' var IPython = (function (IPython) {'
         $([IPython.events]).on('checkpoints_listed.Notebook', function (event, data) {
             that.set_last_checkpoint(data[0]);
         });
-        
+
         $([IPython.events]).on('checkpoint_created.Notebook', function (event, data) {
             that.set_last_checkpoint(data);
         });
@@ -104,7 +104,7 b' var IPython = (function (IPython) {'
                         return false;
                     }
                 });
-                that.find('input[type="text"]').focus();
+                that.find('input[type="text"]').focus().select();
             }
         });
     }

IPython/html/static/notebook/js/textcell.js

@@ -482,6 +482,22 b' var IPython = (function (IPython) {'
         return data;
     };
 
+    /**
+     * can the cell be split into two cells
+     * @method is_splittable
+     **/
+    HeadingCell.prototype.is_splittable = function () {
+        return false;
+    };
+
+
+    /**
+     * can the cell be merged with other cells
+     * @method is_mergeable
+     **/
+    HeadingCell.prototype.is_mergeable = function () {
+        return false;
+    };
 
     /**
      * Change heading level of cell, and re-render

IPython/html/static/notebook/js/tooltip.js

@@ -161,16 +161,23 b' var IPython = (function (IPython) {'
         this.code_mirror = null;
     }
 
+    // return true on successfully removing a visible tooltip; otherwise return
+    // false.
     Tooltip.prototype.remove_and_cancel_tooltip = function (force) {
         // note that we don't handle closing directly inside the calltip
         // as in the completer, because it is not focusable, so won't
         // get the event.
-        if (this._sticky == false || force == true) {
-            this.cancel_stick();
-            this._hide();
+        if (!this._hidden) {
+          if (force || !this._sticky) {
+              this.cancel_stick();
+              this._hide();
+          }
+          this.cancel_pending();
+          this.reset_tabs_function();
+          return true;
+        } else {
+          return false;
         }
-        this.cancel_pending();
-        this.reset_tabs_function();
     }
 
     // cancel autocall done after '(' for example.
@@ -335,7 +342,7 b' var IPython = (function (IPython) {'
         if (docstring == null) {
             docstring = reply.docstring;
         }
-        
+
         if (docstring == null) {
             // For reals this time, no docstring
             if (this._hide_if_no_docstring) {

IPython/html/static/notebook/less/cell.less

@@ -1,4 +1,4 b''
-.cell {
+div.cell {
     border: 1px solid transparent;
     .vbox();
 
@@ -6,9 +6,6 b''
         .corner-all;
         border : thin @border_color solid;
     }
-}
-
-div.cell {
     width: 100%;
     padding: 5px 5px 5px 0px;
     /* This acts as a spacer between cells, that is outside the border */

IPython/html/static/notebook/less/codemirror.less

@@ -22,7 +22,7 b''
     overflow-x: auto;
 }
 
-@-moz-document {
+@-moz-document url-prefix() {
   /* Firefox does weird and terrible things (#3549) when overflow-x is auto */
   /* It doesn't respect the overflow setting anyway, so we can workaround it with this */
   .CodeMirror-scroll {

IPython/html/static/services/kernels/js/kernel.js

@@ -119,7 +119,6 b' var IPython = (function (IPython) {'
         this.ws_url = ws_url;
         this.kernel_url = this.base_url + "/" + this.kernel_id;
         this.start_channels();
-        $([IPython.events]).trigger('status_started.Kernel', {kernel: this});
     };
 
 
@@ -144,11 +143,7 b' var IPython = (function (IPython) {'
         this.shell_channel = new this.WebSocket(ws_url + "/shell");
         this.stdin_channel = new this.WebSocket(ws_url + "/stdin");
         this.iopub_channel = new this.WebSocket(ws_url + "/iopub");
-        send_cookie = function(){
-            // send the session id so the Session object Python-side
-            // has the same identity
-            this.send(that.session_id + ':' + document.cookie);
-        };
+        
         var already_called_onclose = false; // only alert once
         var ws_closed_early = function(evt){
             if (already_called_onclose){
@@ -170,7 +165,7 b' var IPython = (function (IPython) {'
         };
         var channels = [this.shell_channel, this.iopub_channel, this.stdin_channel];
         for (var i=0; i < channels.length; i++) {
-            channels[i].onopen = send_cookie;
+            channels[i].onopen = $.proxy(this._ws_opened, this);
             channels[i].onclose = ws_closed_early;
         }
         // switch from early-close to late-close message after 1s
@@ -187,7 +182,27 b' var IPython = (function (IPython) {'
     };
 
     /**
-     * Stop the `shell`and `iopub` channels.
+     * Handle a websocket entering the open state
+     * sends session and cookie authentication info as first message.
+     * Once all sockets are open, signal the Kernel.status_started event.
+     * @method _ws_opened
+     */
+    Kernel.prototype._ws_opened = function (evt) {
+        // send the session id so the Session object Python-side
+        // has the same identity
+        evt.target.send(this.session_id + ':' + document.cookie);
+        
+        var channels = [this.shell_channel, this.iopub_channel, this.stdin_channel];
+        for (var i=0; i < channels.length; i++) {
+            // if any channel is not ready, don't trigger event.
+            if ( !channels[i].readyState ) return;
+        }
+        // all events ready, trigger started event.
+        $([IPython.events]).trigger('status_started.Kernel', {kernel: this});
+    };
+    
+    /**
+     * Stop the websocket channels.
      * @method stop_channels
      */
     Kernel.prototype.stop_channels = function () {
@@ -388,9 +403,16 b' var IPython = (function (IPython) {'
     };
 
 
+    Kernel.prototype.clear_callbacks_for_msg = function (msg_id) {
+        if (this._msg_callbacks[msg_id] !== undefined ) {
+            delete this._msg_callbacks[msg_id];
+        }
+    };
+
+
     Kernel.prototype.set_callbacks_for_msg = function (msg_id, callbacks) {
         this._msg_callbacks[msg_id] = callbacks || {};
-    }
+    };
 
 
     Kernel.prototype._handle_shell_reply = function (e) {

IPython/html/static/style/ipython.min.css

@@ -18,20 +18,6 b''
 .start{-webkit-box-pack:start;-moz-box-pack:start;box-pack:start;}
 .end{-webkit-box-pack:end;-moz-box-pack:end;box-pack:end;}
 .center{-webkit-box-pack:center;-moz-box-pack:center;box-pack:center;}
-body{background-color:white;position:absolute;left:0px;right:0px;top:0px;bottom:0px;overflow:visible;}
-div#header{display:none;}
-#ipython_notebook{padding-left:16px;}
-#noscript{width:auto;padding-top:16px;padding-bottom:16px;text-align:center;font-size:22px;color:red;font-weight:bold;}
-#ipython_notebook img{font-family:Verdana,"Helvetica Neue",Arial,Helvetica,Geneva,sans-serif;height:24px;text-decoration:none;color:black;}
-#site{width:100%;display:none;}
-.ui-button .ui-button-text{padding:0.2em 0.8em;font-size:77%;}
-input.ui-button{padding:0.3em 0.9em;}
-.navbar span{margin-top:3px;}
-span#login_widget{float:right;}
-.nav-header{text-transform:none;}
-.navbar-nobg{background-color:transparent;background-image:none;}
-#header>span{margin-top:10px;}
-.modal-body{max-height:500px;}
 .center-nav{display:inline-block;margin-bottom:-4px;}
 .alternate_upload{background-color:none;display:inline;}
 .alternate_upload.form{padding:0;margin:0;}
@@ -67,8 +53,7 b' input.engine_num_input{height:20px;margin-bottom:2px;padding-top:0;padding-botto'
 .ansibgpurple{background-color:magenta;}
 .ansibgcyan{background-color:cyan;}
 .ansibggray{background-color:gray;}
-.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;}.cell.selected{border-radius:4px;border:thin #ababab solid;}
-div.cell{width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;}
+div.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;}div.cell.selected{border-radius:4px;border:thin #ababab solid;}
 div.prompt{width:11ex;padding:0.4em;margin:0px;font-family:monospace;text-align:right;line-height:1.231em;}
 .celltoolbar{border:thin solid #CFCFCF;border-bottom:none;background:#EEE;border-top-right-radius:3px;border-top-left-radius:3px;width:100%;-webkit-box-pack:end;height:22px;}
 .no_input_radius{border-top-right-radius:0px;border-top-left-radius:0px;}
@@ -98,7 +83,7 b' div.out_prompt_overlay:hover{-webkit-box-shadow:inset 0 0 1px #000000;-moz-box-s'
 div.output_prompt{color:darkred;}
 .CodeMirror{line-height:1.231em;height:auto;background:none;}
 .CodeMirror-scroll{overflow-y:hidden;overflow-x:auto;}
-@-moz-document {.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;}
+@-moz-document url-prefix(){.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;}
 .CodeMirror-linenumber{padding:0 8px 0 4px;}
 .CodeMirror-gutters{border-bottom-left-radius:4px;border-top-left-radius:4px;}
 .CodeMirror pre{padding:0;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;}

IPython/html/static/style/style.less

@@ -8,6 +8,7 b''
 
 // base
 @import "../base/less/style.less";
+@import "../base/less/page.less";
 
 // auth
 @import "../auth/less/style.less";

IPython/html/static/style/style.min.css

@@ -1434,8 +1434,7 b' input.engine_num_input{height:20px;margin-bottom:2px;padding-top:0;padding-botto'
 .ansibgpurple{background-color:magenta;}
 .ansibgcyan{background-color:cyan;}
 .ansibggray{background-color:gray;}
-.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;}.cell.selected{border-radius:4px;border:thin #ababab solid;}
-div.cell{width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;}
+div.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;}div.cell.selected{border-radius:4px;border:thin #ababab solid;}
 div.prompt{width:11ex;padding:0.4em;margin:0px;font-family:monospace;text-align:right;line-height:1.231em;}
 .celltoolbar{border:thin solid #CFCFCF;border-bottom:none;background:#EEE;border-top-right-radius:3px;border-top-left-radius:3px;width:100%;-webkit-box-pack:end;height:22px;}
 .no_input_radius{border-top-right-radius:0px;border-top-left-radius:0px;}
@@ -1465,7 +1464,7 b' div.out_prompt_overlay:hover{-webkit-box-shadow:inset 0 0 1px #000000;-moz-box-s'
 div.output_prompt{color:darkred;}
 .CodeMirror{line-height:1.231em;height:auto;background:none;}
 .CodeMirror-scroll{overflow-y:hidden;overflow-x:auto;}
-@-moz-document {.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;}
+@-moz-document url-prefix(){.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;}
 .CodeMirror-linenumber{padding:0 8px 0 4px;}
 .CodeMirror-gutters{border-bottom-left-radius:4px;border-top-left-radius:4px;}
 .CodeMirror pre{padding:0;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;}

IPython/kernel/connect.py

@@ -34,14 +34,13 b' import zmq'
 from IPython.external.ssh import tunnel
 
 # IPython imports
-# from IPython.config import Configurable
+from IPython.config import Configurable
 from IPython.core.profiledir import ProfileDir
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils.path import filefind, get_ipython_dir
 from IPython.utils.py3compat import str_to_bytes, bytes_to_str
 from IPython.utils.traitlets import (
     Bool, Integer, Unicode, CaselessStrEnum,
-    HasTraits,
 )
 
 
@@ -50,7 +49,7 b' from IPython.utils.traitlets import ('
 #-----------------------------------------------------------------------------
 
 def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
-                         control_port=0, ip=LOCALHOST, key=b'', transport='tcp',
+                         control_port=0, ip='', key=b'', transport='tcp',
                          signature_scheme='hmac-sha256',
                          ):
     """Generates a JSON config file, including the selection of random ports.
@@ -91,6 +90,8 b' def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, '
         and 'sha256' is the default hash function.
 
     """
+    if not ip:
+        ip = localhost()
     # default to temporary connector file
     if not fname:
         fname = tempfile.mktemp('.json')
@@ -383,7 +384,7 b' channel_socket_types = {'
 
 port_names = [ "%s_port" % channel for channel in ('shell', 'stdin', 'iopub', 'hb', 'control')]
 
-class ConnectionFileMixin(HasTraits):
+class ConnectionFileMixin(Configurable):
     """Mixin for configurable classes that work with connection files"""
 
     # The addresses for the communication channels
@@ -392,7 +393,7 b' class ConnectionFileMixin(HasTraits):'
 
     transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True)
 
-    ip = Unicode(LOCALHOST, config=True,
+    ip = Unicode(config=True,
         help="""Set the kernel\'s IP address [default localhost].
         If the IP address is something other than localhost, then
         Consoles on other machines will be able to connect
@@ -406,7 +407,7 b' class ConnectionFileMixin(HasTraits):'
             else:
                 return 'kernel-ipc'
         else:
-            return LOCALHOST
+            return localhost()
 
     def _ip_changed(self, name, old, new):
         if new == '*':

IPython/kernel/manager.py

@@ -1,5 +1,4 b''
-"""Base class to manage a running kernel
-"""
+"""Base class to manage a running kernel"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2013  The IPython Development Team
@@ -24,7 +23,7 b' import zmq'
 # Local imports
 from IPython.config.configurable import LoggingConfigurable
 from IPython.utils.importstring import import_item
-from IPython.utils.localinterfaces import LOCAL_IPS
+from IPython.utils.localinterfaces import is_local_ip, local_ips
 from IPython.utils.traitlets import (
     Any, Instance, Unicode, List, Bool, Type, DottedObjectName
 )
@@ -185,11 +184,11 b' class KernelManager(LoggingConfigurable, ConnectionFileMixin):'
              keyword arguments that are passed down to build the kernel_cmd
              and launching the kernel (e.g. Popen kwargs).
         """
-        if self.transport == 'tcp' and self.ip not in LOCAL_IPS:
+        if self.transport == 'tcp' and not is_local_ip(self.ip):
             raise RuntimeError("Can only launch a kernel on a local interface. "
                                "Make sure that the '*_address' attributes are "
                                "configured properly. "
-                               "Currently valid addresses are: %s"%LOCAL_IPS
+                               "Currently valid addresses are: %s" % local_ips()
                                )
 
         # write connection file / get default ports

IPython/kernel/multikernelmanager.py

@@ -171,7 +171,7 b' class MultiKernelManager(LoggingConfigurable):'
         self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum))
 
     @kernel_method
-    def restart_kernel(self, kernel_id):
+    def restart_kernel(self, kernel_id, now=False):
         """Restart a kernel by its uuid, keeping the same ports.
 
         Parameters

IPython/kernel/tests/test_kernel.py

@@ -11,101 +11,18 b''
 # Imports
 #-------------------------------------------------------------------------------
 
-import os
-import shutil
 import sys
-import tempfile
-
-from contextlib import contextmanager
-from subprocess import PIPE
 
 import nose.tools as nt
 
-from IPython.kernel import KernelManager
-from IPython.kernel.tests.test_message_spec import execute, flush_channels
-from IPython.testing import decorators as dec
-from IPython.utils import path
+from IPython.testing import decorators as dec, tools as tt
+from IPython.utils import py3compat
+
+from .utils import new_kernel, kernel, TIMEOUT, assemble_output, execute, flush_channels
 
 #-------------------------------------------------------------------------------
 # Tests
 #-------------------------------------------------------------------------------
-IPYTHONDIR = None
-save_env = None
-save_get_ipython_dir = None
-
-def setup():
-    """setup temporary IPYTHONDIR for tests"""
-    global IPYTHONDIR
-    global save_env
-    global save_get_ipython_dir
-    
-    IPYTHONDIR = tempfile.mkdtemp()
-
-    save_env = os.environ.copy()
-    os.environ["IPYTHONDIR"] = IPYTHONDIR
-
-    save_get_ipython_dir = path.get_ipython_dir
-    path.get_ipython_dir = lambda : IPYTHONDIR
-
-
-def teardown():
-    path.get_ipython_dir = save_get_ipython_dir
-    os.environ = save_env
-    
-    try:
-        shutil.rmtree(IPYTHONDIR)
-    except (OSError, IOError):
-        # no such file
-        pass
-
-
-@contextmanager
-def new_kernel():
-    """start a kernel in a subprocess, and wait for it to be ready
-    
-    Returns
-    -------
-    kernel_manager: connected KernelManager instance
-    """
-    KM = KernelManager()
-
-    KM.start_kernel(stdout=PIPE, stderr=PIPE)
-    KC = KM.client()
-    KC.start_channels()
-    
-    # wait for kernel to be ready
-    KC.shell_channel.execute("import sys")
-    KC.shell_channel.get_msg(block=True, timeout=5)
-    flush_channels(KC)
-    try:
-        yield KC
-    finally:
-        KC.stop_channels()
-        KM.shutdown_kernel()
-
-
-def assemble_output(iopub):
-    """assemble stdout/err from an execution"""
-    stdout = ''
-    stderr = ''
-    while True:
-        msg = iopub.get_msg(block=True, timeout=1)
-        msg_type = msg['msg_type']
-        content = msg['content']
-        if msg_type == 'status' and content['execution_state'] == 'idle':
-            # idle message signals end of output
-            break
-        elif msg['msg_type'] == 'stream':
-            if content['name'] == 'stdout':
-                stdout = stdout + content['data']
-            elif content['name'] == 'stderr':
-                stderr = stderr + content['data']
-            else:
-                raise KeyError("bad stream: %r" % content['name'])
-        else:
-            # other output, ignored
-            pass
-    return stdout, stderr
 
 
 def _check_mp_mode(kc, expected=False, stream="stdout"):
@@ -116,16 +33,17 b' def _check_mp_mode(kc, expected=False, stream="stdout"):'
     nt.assert_equal(eval(stdout.strip()), expected)
 
 
+# printing tests
+
 def test_simple_print():
     """simple print statement in kernel"""
-    with new_kernel() as kc:
+    with kernel() as kc:
         iopub = kc.iopub_channel
         msg_id, content = execute(kc=kc, code="print ('hi')")
         stdout, stderr = assemble_output(iopub)
         nt.assert_equal(stdout, 'hi\n')
         nt.assert_equal(stderr, '')
         _check_mp_mode(kc, expected=False)
-    print ('hello')
 
 
 @dec.knownfailureif(sys.platform == 'win32', "subprocess prints fail on Windows")
@@ -161,7 +79,7 b' def test_subprocess_print():'
 
 def test_subprocess_noprint():
     """mp.Process without print doesn't trigger iostream mp_mode"""
-    with new_kernel() as kc:
+    with kernel() as kc:
         iopub = kc.iopub_channel
         
         np = 5
@@ -202,3 +120,51 b' def test_subprocess_error():'
         _check_mp_mode(kc, expected=False)
         _check_mp_mode(kc, expected=False, stream="stderr")
 
+# raw_input tests
+
+def test_raw_input():
+    """test [raw_]input"""
+    with kernel() as kc:
+        iopub = kc.iopub_channel
+        
+        input_f = "input" if py3compat.PY3 else "raw_input"
+        theprompt = "prompt> "
+        code = 'print({input_f}("{theprompt}"))'.format(**locals())
+        msg_id = kc.execute(code, allow_stdin=True)
+        msg = kc.get_stdin_msg(block=True, timeout=TIMEOUT)
+        nt.assert_equal(msg['header']['msg_type'], u'input_request')
+        content = msg['content']
+        nt.assert_equal(content['prompt'], theprompt)
+        text = "some text"
+        kc.input(text)
+        reply = kc.get_shell_msg(block=True, timeout=TIMEOUT)
+        nt.assert_equal(reply['content']['status'], 'ok')
+        stdout, stderr = assemble_output(iopub)
+        nt.assert_equal(stdout, text + "\n")
+
+
[e-mail](py3compat.PY3)
+def test_eval_input():
+    """test input() on Python 2"""
+    with kernel() as kc:
+        iopub = kc.iopub_channel
+        
+        input_f = "input" if py3compat.PY3 else "raw_input"
+        theprompt = "prompt> "
+        code = 'print(input("{theprompt}"))'.format(**locals())
+        msg_id = kc.execute(code, allow_stdin=True)
+        msg = kc.get_stdin_msg(block=True, timeout=TIMEOUT)
+        nt.assert_equal(msg['header']['msg_type'], u'input_request')
+        content = msg['content']
+        nt.assert_equal(content['prompt'], theprompt)
+        kc.input("1+1")
+        reply = kc.get_shell_msg(block=True, timeout=TIMEOUT)
+        nt.assert_equal(reply['content']['status'], 'ok')
+        stdout, stderr = assemble_output(iopub)
+        nt.assert_equal(stdout, "2\n")
+
+
+def test_help_output():
+    """ipython kernel --help-all works"""
+    tt.help_all_output_test('kernel')
+

IPython/kernel/tests/test_kernelmanager.py

@@ -26,18 +26,11 b' class TestKernelManager(TestCase):'
     def _run_lifecycle(self, km):
         km.start_kernel(stdout=PIPE, stderr=PIPE)
         self.assertTrue(km.is_alive())
-        km.restart_kernel()
+        km.restart_kernel(now=True)
         self.assertTrue(km.is_alive())
-        # We need a delay here to give the restarting kernel a chance to
-        # restart. Otherwise, the interrupt will kill it, causing the test
-        # suite to hang. The reason it *hangs* is that the shutdown
-        # message for the restart sometimes hasn't been sent to the kernel.
-        # Because linger is oo on the shell channel, the context can't
-        # close until the message is sent to the kernel, which is not dead.
-        time.sleep(1.0)
         km.interrupt_kernel()
         self.assertTrue(isinstance(km, KernelManager))
-        km.shutdown_kernel()
+        km.shutdown_kernel(now=True)
 
     def test_tcp_lifecycle(self):
         km = self._get_tcp_km()

IPython/kernel/tests/test_launcher.py

@@ -20,14 +20,12 b' Authors'
 import nose.tools as nt
 
 # Our own imports
-from IPython.testing import decorators as dec
 from IPython.kernel.launcher import swallow_argv
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
[e-mail]
 def test_swallow_argv():
     tests = [
         # expected  , argv       , aliases, flags
@@ -56,5 +54,5 b' def test_swallow_argv():'
             "expected : %r" % expected,
             "returned : %r" % stripped,
         ])
-        yield nt.assert_equal(expected, stripped, message)
+        nt.assert_equal(expected, stripped, message)
 

IPython/kernel/tests/test_message_spec.py

@@ -1,7 +1,7 b''
-"""Test suite for our zeromq-based messaging specification.
+"""Test suite for our zeromq-based message specification.
 """
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2010-2011  The IPython Development Team
+#  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING.txt, distributed as part of this software.
@@ -15,77 +15,25 b' import nose.tools as nt'
 
 from IPython.kernel import KernelManager
 
-from IPython.testing import decorators as dec
 from IPython.utils.traitlets import (
     HasTraits, TraitError, Bool, Unicode, Dict, Integer, List, Enum, Any,
 )
 
+from .utils import TIMEOUT, start_global_kernel, flush_channels, execute
+
 #-----------------------------------------------------------------------------
-# Global setup and utilities
+# Globals
 #-----------------------------------------------------------------------------
-
-STARTUP_TIMEOUT = 60
-TIMEOUT = 15
+KC = None
 
 def setup():
-    global KM, KC
-    KM = KernelManager()
-    KM.start_kernel(stdout=PIPE, stderr=PIPE)
-    KC = KM.client()
-    KC.start_channels()
-    
-    # wait for kernel to be ready
-    try:
-        msg = KC.iopub_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT)
-    except Empty:
-        pass
-    msg_id = KC.kernel_info()
-    KC.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT)
-    flush_channels()
-
-
-def teardown():
-    KC.stop_channels()
-    KM.shutdown_kernel()
-
-
-def flush_channels(kc=None):
-    """flush any messages waiting on the queue"""
-    if kc is None:
-        kc = KC
-    for channel in (kc.shell_channel, kc.iopub_channel):
-        while True:
-            try:
-                msg = channel.get_msg(block=True, timeout=0.1)
-            except Empty:
-                break
-            else:
-                list(validate_message(msg))
-
-
-def execute(code='', kc=None, **kwargs):
-    """wrapper for doing common steps for validating an execution request"""
-    if kc is None:
-        kc = KC
-    msg_id = kc.execute(code=code, **kwargs)
-    reply = kc.get_shell_msg(timeout=TIMEOUT)
-    list(validate_message(reply, 'execute_reply', msg_id))
-    busy = kc.get_iopub_msg(timeout=TIMEOUT)
-    list(validate_message(busy, 'status', msg_id))
-    nt.assert_equal(busy['content']['execution_state'], 'busy')
-    
-    if not kwargs.get('silent'):
-        pyin = kc.get_iopub_msg(timeout=TIMEOUT)
-        list(validate_message(pyin, 'pyin', msg_id))
-        nt.assert_equal(pyin['content']['code'], code)
-    
-    return msg_id, reply['content']
+    global KC
+    KC = start_global_kernel()
 
 #-----------------------------------------------------------------------------
-# MSG Spec References
+# Message Spec References
 #-----------------------------------------------------------------------------
 
-
 class Reference(HasTraits):
 
     """
@@ -101,14 +49,14 b' class Reference(HasTraits):'
     def check(self, d):
         """validate a dict against our traits"""
         for key in self.trait_names():
-            yield nt.assert_true(key in d, "Missing key: %r, should be found in %s" % (key, d))
+            nt.assert_in(key, d)
             # FIXME: always allow None, probably not a good idea
             if d[key] is None:
                 continue
             try:
                 setattr(self, key, d[key])
             except TraitError as e:
-                yield nt.assert_true(False, str(e))
+                nt.assert_true(False, str(e))
 
 
 class RMessage(Reference):
@@ -133,14 +81,11 b' class ExecuteReply(Reference):'
     status = Enum((u'ok', u'error'))
     
     def check(self, d):
-        for tst in Reference.check(self, d):
-            yield tst
+        Reference.check(self, d)
         if d['status'] == 'ok':
-            for tst in ExecuteReplyOkay().check(d):
-                yield tst
+            ExecuteReplyOkay().check(d)
         elif d['status'] == 'error':
-            for tst in ExecuteReplyError().check(d):
-                yield tst
+            ExecuteReplyError().check(d)
 
 
 class ExecuteReplyOkay(Reference):
@@ -177,11 +122,9 b' class OInfoReply(Reference):'
     source = Unicode()
     
     def check(self, d):
-        for tst in Reference.check(self, d):
-            yield tst
+        Reference.check(self, d)
         if d['argspec'] is not None:
-            for tst in ArgSpec().check(d['argspec']):
-                yield tst
+            ArgSpec().check(d['argspec'])
 
 
 class ArgSpec(Reference):
@@ -212,10 +155,8 b' class KernelInfoReply(Reference):'
 
     def _ipython_version_changed(self, name, old, new):
         for v in new:
-            nt.assert_true(
-                isinstance(v, int) or isinstance(v, basestring),
-                'expected int or string as version component, got {0!r}'
-                .format(v))
+            assert isinstance(v, int) or isinstance(v, basestring), \
+            'expected int or string as version component, got {0!r}'.format(v)
 
 
 # IOPub messages
@@ -241,8 +182,8 b' class DisplayData(Reference):'
     data = Dict()
     def _data_changed(self, name, old, new):
         for k,v in new.iteritems():
-            nt.assert_true(mime_pat.match(k))
-            nt.assert_true(isinstance(v, basestring), "expected string data, got %r" % v)
+            assert mime_pat.match(k)
+            nt.assert_is_instance(v, basestring)
 
 
 class PyOut(Reference):
@@ -250,8 +191,8 b' class PyOut(Reference):'
     data = Dict()
     def _data_changed(self, name, old, new):
         for k,v in new.iteritems():
-            nt.assert_true(mime_pat.match(k))
-            nt.assert_true(isinstance(v, basestring), "expected string data, got %r" % v)
+            assert mime_pat.match(k)
+            nt.assert_is_instance(v, basestring)
 
 
 references = {
@@ -282,13 +223,12 b' def validate_message(msg, msg_type=None, parent=None):'
     """
     RMessage().check(msg)
     if msg_type:
-        yield nt.assert_equal(msg['msg_type'], msg_type)
+        nt.assert_equal(msg['msg_type'], msg_type)
     if parent:
-        yield nt.assert_equal(msg['parent_header']['msg_id'], parent)
+        nt.assert_equal(msg['parent_header']['msg_id'], parent)
     content = msg['content']
     ref = references[msg['msg_type']]
-    for tst in ref.check(content):
-        yield tst
+    ref.check(content)
 
 
 #-----------------------------------------------------------------------------
@@ -297,54 +237,47 b' def validate_message(msg, msg_type=None, parent=None):'
 
 # Shell channel
 
[e-mail]
 def test_execute():
     flush_channels()
     
     msg_id = KC.execute(code='x=1')
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'execute_reply', msg_id):
-        yield tst
+    validate_message(reply, 'execute_reply', msg_id)
 
 
[e-mail]
 def test_execute_silent():
     flush_channels()
     msg_id, reply = execute(code='x=1', silent=True)
     
     # flush status=idle
     status = KC.iopub_channel.get_msg(timeout=TIMEOUT)
-    for tst in validate_message(status, 'status', msg_id):
-        yield tst
+    validate_message(status, 'status', msg_id)
     nt.assert_equal(status['content']['execution_state'], 'idle')
 
-    yield nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
+    nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
     count = reply['execution_count']
     
     msg_id, reply = execute(code='x=2', silent=True)
     
     # flush status=idle
     status = KC.iopub_channel.get_msg(timeout=TIMEOUT)
-    for tst in validate_message(status, 'status', msg_id):
-        yield tst
-    yield nt.assert_equal(status['content']['execution_state'], 'idle')
+    validate_message(status, 'status', msg_id)
+    nt.assert_equal(status['content']['execution_state'], 'idle')
     
-    yield nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
+    nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1)
     count_2 = reply['execution_count']
-    yield nt.assert_equal(count_2, count)
+    nt.assert_equal(count_2, count)
 
 
[e-mail]
 def test_execute_error():
     flush_channels()
     
     msg_id, reply = execute(code='1/0')
-    yield nt.assert_equal(reply['status'], 'error')
-    yield nt.assert_equal(reply['ename'], 'ZeroDivisionError')
+    nt.assert_equal(reply['status'], 'error')
+    nt.assert_equal(reply['ename'], 'ZeroDivisionError')
     
     pyerr = KC.iopub_channel.get_msg(timeout=TIMEOUT)
-    for tst in validate_message(pyerr, 'pyerr', msg_id):
-        yield tst
+    validate_message(pyerr, 'pyerr', msg_id)
 
 
 def test_execute_inc():
@@ -405,17 +338,14 b' def test_user_expressions_fail():'
     nt.assert_equal(foo['ename'], 'NameError')
 
 
[e-mail]
 def test_oinfo():
     flush_channels()
 
     msg_id = KC.object_info('a')
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'object_info_reply', msg_id):
-        yield tst
+    validate_message(reply, 'object_info_reply', msg_id)
 
 
[e-mail]
 def test_oinfo_found():
     flush_channels()
 
@@ -423,15 +353,13 b' def test_oinfo_found():'
     
     msg_id = KC.object_info('a')
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'object_info_reply', msg_id):
-        yield tst
+    validate_message(reply, 'object_info_reply', msg_id)
     content = reply['content']
-    yield nt.assert_true(content['found'])
+    assert content['found']
     argspec = content['argspec']
-    yield nt.assert_true(argspec is None, "didn't expect argspec dict, got %r" % argspec)
+    nt.assert_is(argspec, None)
 
 
[e-mail]
 def test_oinfo_detail():
     flush_channels()
 
@@ -439,28 +367,24 b' def test_oinfo_detail():'
     
     msg_id = KC.object_info('ip.object_inspect', detail_level=2)
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'object_info_reply', msg_id):
-        yield tst
+    validate_message(reply, 'object_info_reply', msg_id)
     content = reply['content']
-    yield nt.assert_true(content['found'])
+    assert content['found']
     argspec = content['argspec']
-    yield nt.assert_true(isinstance(argspec, dict), "expected non-empty argspec dict, got %r" % argspec)
-    yield nt.assert_equal(argspec['defaults'], [0])
+    nt.assert_is_instance(argspec, dict, "expected non-empty argspec dict, got %r" % argspec)
+    nt.assert_equal(argspec['defaults'], [0])
 
 
[e-mail]
 def test_oinfo_not_found():
     flush_channels()
 
     msg_id = KC.object_info('dne')
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'object_info_reply', msg_id):
-        yield tst
+    validate_message(reply, 'object_info_reply', msg_id)
     content = reply['content']
-    yield nt.assert_false(content['found'])
+    nt.assert_false(content['found'])
 
 
[e-mail]
 def test_complete():
     flush_channels()
 
@@ -468,49 +392,42 b' def test_complete():'
     
     msg_id = KC.complete('al', 'al', 2)
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'complete_reply', msg_id):
-        yield tst
+    validate_message(reply, 'complete_reply', msg_id)
     matches = reply['content']['matches']
     for name in ('alpha', 'albert'):
-        yield nt.assert_true(name in matches, "Missing match: %r" % name)
+        nt.assert_in(name, matches)
 
 
[e-mail]
 def test_kernel_info_request():
     flush_channels()
 
     msg_id = KC.kernel_info()
     reply = KC.get_shell_msg(timeout=TIMEOUT)
-    for tst in validate_message(reply, 'kernel_info_reply', msg_id):
-        yield tst
+    validate_message(reply, 'kernel_info_reply', msg_id)
 
 
 # IOPub channel
 
 
[e-mail]
 def test_stream():
     flush_channels()
 
     msg_id, reply = execute("print('hi')")
 
     stdout = KC.iopub_channel.get_msg(timeout=TIMEOUT)
-    for tst in validate_message(stdout, 'stream', msg_id):
-        yield tst
+    validate_message(stdout, 'stream', msg_id)
     content = stdout['content']
-    yield nt.assert_equal(content['name'], u'stdout')
-    yield nt.assert_equal(content['data'], u'hi\n')
+    nt.assert_equal(content['name'], u'stdout')
+    nt.assert_equal(content['data'], u'hi\n')
 
 
[e-mail]
 def test_display_data():
     flush_channels()
 
     msg_id, reply = execute("from IPython.core.display import display; display(1)")
     
     display = KC.iopub_channel.get_msg(timeout=TIMEOUT)
-    for tst in validate_message(display, 'display_data', parent=msg_id):
-        yield tst
+    validate_message(display, 'display_data', parent=msg_id)
     data = display['content']['data']
-    yield nt.assert_equal(data['text/plain'], u'1')
+    nt.assert_equal(data['text/plain'], u'1')
 

IPython/kernel/tests/test_multikernelmanager.py

@@ -7,7 +7,7 b' from unittest import TestCase'
 from IPython.testing import decorators as dec
 
 from IPython.config.loader import Config
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.kernel import KernelManager
 from IPython.kernel.multikernelmanager import MultiKernelManager
 
@@ -31,20 +31,13 b' class TestKernelManager(TestCase):'
         self.assertTrue(kid in km)
         self.assertTrue(kid in km.list_kernel_ids())
         self.assertEqual(len(km),1)
-        km.restart_kernel(kid)
+        km.restart_kernel(kid, now=True)
         self.assertTrue(km.is_alive(kid))
         self.assertTrue(kid in km.list_kernel_ids())
-        # We need a delay here to give the restarting kernel a chance to
-        # restart. Otherwise, the interrupt will kill it, causing the test
-        # suite to hang. The reason it *hangs* is that the shutdown
-        # message for the restart sometimes hasn't been sent to the kernel.
-        # Because linger is oo on the shell channel, the context can't
-        # close until the message is sent to the kernel, which is not dead.
-        time.sleep(1.0)
         km.interrupt_kernel(kid)
         k = km.get_kernel(kid)
         self.assertTrue(isinstance(k, KernelManager))
-        km.shutdown_kernel(kid)
+        km.shutdown_kernel(kid, now=True)
         self.assertTrue(not kid in km)
 
     def _run_cinfo(self, km, transport, ip):
@@ -63,7 +56,7 b' class TestKernelManager(TestCase):'
         self.assertTrue('hb_port' in cinfo)
         stream = km.connect_hb(kid)
         stream.close()
-        km.shutdown_kernel(kid)
+        km.shutdown_kernel(kid, now=True)
 
     def test_tcp_lifecycle(self):
         km = self._get_tcp_km()
@@ -71,7 +64,7 b' class TestKernelManager(TestCase):'
     
     def test_tcp_cinfo(self):
         km = self._get_tcp_km()
-        self._run_cinfo(km, 'tcp', LOCALHOST)
+        self._run_cinfo(km, 'tcp', localhost())
 
     @dec.skip_win32
     def test_ipc_lifecycle(self):

IPython/kernel/tests/test_public_api.py

@@ -14,8 +14,6 b' Authors'
 
 import nose.tools as nt
 
-from IPython.testing import decorators as dec
-
 from IPython.kernel import launcher, connect
 from IPython import kernel
 
@@ -23,25 +21,21 b' from IPython import kernel'
 # Classes and functions
 #-----------------------------------------------------------------------------
 
[e-mail]
 def test_kms():
     for base in ("", "Multi"):
         KM = base + "KernelManager"
-        yield nt.assert_true(KM in dir(kernel), KM)
+        nt.assert_in(KM, dir(kernel))
 
[e-mail]
 def test_kcs():
     for base in ("", "Blocking"):
         KM = base + "KernelClient"
-        yield nt.assert_true(KM in dir(kernel), KM)
+        nt.assert_in(KM, dir(kernel))
 
[e-mail]
 def test_launcher():
     for name in launcher.__all__:
-        yield nt.assert_true(name in dir(kernel), name)
+        nt.assert_in(name, dir(kernel))
 
[e-mail]
 def test_connect():
     for name in connect.__all__:
-        yield nt.assert_true(name in dir(kernel), name)
+        nt.assert_in(name, dir(kernel))
 

IPython/kernel/zmq/heartbeat.py

@@ -19,7 +19,7 b' from threading import Thread'
 
 import zmq
 
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 
 #-----------------------------------------------------------------------------
 # Code
@@ -29,7 +29,9 b' from IPython.utils.localinterfaces import LOCALHOST'
 class Heartbeat(Thread):
     "A simple ping-pong style heartbeat that runs in a thread."
 
-    def __init__(self, context, addr=('tcp', LOCALHOST, 0)):
+    def __init__(self, context, addr=None):
+        if addr is None:
+            addr = ('tcp', localhost(), 0)
         Thread.__init__(self)
         self.context = context
         self.transport, self.ip, self.port = addr

IPython/kernel/zmq/ipkernel.py

@@ -87,7 +87,7 b' class Kernel(Configurable):'
         if self.shell is not None:
             self.shell.user_module = new
     
-    user_ns = Dict(default_value=None)
+    user_ns = Instance(dict, args=None, allow_none=True)
     def _user_ns_changed(self, name, old, new):
         if self.shell is not None:
             self.shell.user_ns = new
@@ -131,6 +131,7 b' class Kernel(Configurable):'
     # A reference to the Python builtin 'raw_input' function.
     # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3)
     _sys_raw_input = Any()
+    _sys_eval_input = Any()
 
     # set of aborted msg_ids
     aborted = Set()
@@ -352,15 +353,18 b' class Kernel(Configurable):'
         # raw_input in the user namespace.
         if content.get('allow_stdin', False):
             raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
+            input = lambda prompt='': eval(raw_input(prompt))
         else:
-            raw_input = lambda prompt='' : self._no_raw_input()
+            raw_input = input = lambda prompt='' : self._no_raw_input()
 
         if py3compat.PY3:
             self._sys_raw_input = __builtin__.input
             __builtin__.input = raw_input
         else:
             self._sys_raw_input = __builtin__.raw_input
+            self._sys_eval_input = __builtin__.input
             __builtin__.raw_input = raw_input
+            __builtin__.input = input
 
         # Set the parent message of the display hook and out streams.
         shell.displayhook.set_parent(parent)
@@ -403,6 +407,7 b' class Kernel(Configurable):'
                  __builtin__.input = self._sys_raw_input
              else:
                  __builtin__.raw_input = self._sys_raw_input
+                 __builtin__.input = self._sys_eval_input
 
         reply_content[u'status'] = status
 

IPython/kernel/zmq/kernelapp.py

@@ -39,7 +39,7 b' from IPython.core.shellapp import ('
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.utils import io
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils.path import filefind
 from IPython.utils.py3compat import str_to_bytes
 from IPython.utils.traitlets import (
@@ -156,7 +156,8 b' class IPKernelApp(BaseIPythonApplication, InteractiveShellApp):'
             else:
                 return 'kernel-ipc'
         else:
-            return LOCALHOST
+            return localhost()
+    
     hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]")
     shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]")
     iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]")

IPython/kernel/zmq/session.py

@@ -436,8 +436,15 b' class Session(Configurable):'
         msg = dict(a=[1,'hi'])
         try:
             packed = pack(msg)
-        except Exception:
-            raise ValueError("packer could not serialize a simple message")
+        except Exception as e:
+            msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}"
+            if self.packer == 'json':
+                jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
+            else:
+                jsonmsg = ""
+            raise ValueError(
+                msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg)
+            )
 
         # ensure packed message is bytes
         if not isinstance(packed, bytes):
@@ -446,8 +453,16 b' class Session(Configurable):'
         # check that unpack is pack's inverse
         try:
             unpacked = unpack(packed)
-        except Exception:
-            raise ValueError("unpacker could not handle the packer's output")
+            assert unpacked == msg
+        except Exception as e:
+            msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}"
+            if self.packer == 'json':
+                jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod
+            else:
+                jsonmsg = ""
+            raise ValueError(
+                msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg)
+            )
 
         # check datetime support
         msg = dict(t=datetime.now())

IPython/kernel/zmq/tests/test_serialize.py

@@ -46,7 +46,6 b" DTYPES = ('uint8', 'float64', 'int32', [('g', 'float32')], '|S10')"
 # Tests
 #-------------------------------------------------------------------------------
 
[e-mail]
 def test_roundtrip_simple():
     for obj in [
         'hello',
@@ -55,18 +54,16 b' def test_roundtrip_simple():'
         (b'123', 'hello'),
     ]:
         obj2 = roundtrip(obj)
-        yield nt.assert_equals(obj, obj2)
+        nt.assert_equal(obj, obj2)
 
[e-mail]
 def test_roundtrip_nested():
     for obj in [
         dict(a=range(5), b={1:b'hello'}),
         [range(5),[range(3),(1,[b'whoda'])]],
     ]:
         obj2 = roundtrip(obj)
-        yield nt.assert_equals(obj, obj2)
+        nt.assert_equal(obj, obj2)
 
[e-mail]
 def test_roundtrip_buffered():
     for obj in [
         dict(a=b"x"*1025),
@@ -74,10 +71,10 b' def test_roundtrip_buffered():'
         [b"hello"*501, 1,2,3]
     ]:
         bufs = serialize_object(obj)
-        yield nt.assert_equals(len(bufs), 2)
+        nt.assert_equal(len(bufs), 2)
         obj2, remainder = unserialize_object(bufs)
-        yield nt.assert_equals(remainder, [])
-        yield nt.assert_equals(obj, obj2)
+        nt.assert_equal(remainder, [])
+        nt.assert_equal(obj, obj2)
 
 def _scrub_nan(A):
     """scrub nans out of empty arrays
@@ -93,7 +90,6 b' def _scrub_nan(A):'
                 # e.g. str dtype
                 pass
 
[e-mail]
 @dec.skip_without('numpy')
 def test_numpy():
     import numpy
@@ -104,12 +100,11 b' def test_numpy():'
             _scrub_nan(A)
             bufs = serialize_object(A)
             B, r = unserialize_object(bufs)
-            yield nt.assert_equals(r, [])
-            yield nt.assert_equals(A.shape, B.shape)
-            yield nt.assert_equals(A.dtype, B.dtype)
-            yield assert_array_equal(A,B)
+            nt.assert_equal(r, [])
+            nt.assert_equal(A.shape, B.shape)
+            nt.assert_equal(A.dtype, B.dtype)
+            assert_array_equal(A,B)
 
[e-mail]
 @dec.skip_without('numpy')
 def test_recarray():
     import numpy
@@ -124,12 +119,11 b' def test_recarray():'
             
             bufs = serialize_object(A)
             B, r = unserialize_object(bufs)
-            yield nt.assert_equals(r, [])
-            yield nt.assert_equals(A.shape, B.shape)
-            yield nt.assert_equals(A.dtype, B.dtype)
-            yield assert_array_equal(A,B)
+            nt.assert_equal(r, [])
+            nt.assert_equal(A.shape, B.shape)
+            nt.assert_equal(A.dtype, B.dtype)
+            assert_array_equal(A,B)
 
[e-mail]
 @dec.skip_without('numpy')
 def test_numpy_in_seq():
     import numpy
@@ -140,15 +134,14 b' def test_numpy_in_seq():'
             _scrub_nan(A)
             bufs = serialize_object((A,1,2,b'hello'))
             canned = pickle.loads(bufs[0])
-            yield nt.assert_true(canned[0], CannedArray)
+            nt.assert_is_instance(canned[0], CannedArray)
             tup, r = unserialize_object(bufs)
             B = tup[0]
-            yield nt.assert_equals(r, [])
-            yield nt.assert_equals(A.shape, B.shape)
-            yield nt.assert_equals(A.dtype, B.dtype)
-            yield assert_array_equal(A,B)
+            nt.assert_equal(r, [])
+            nt.assert_equal(A.shape, B.shape)
+            nt.assert_equal(A.dtype, B.dtype)
+            assert_array_equal(A,B)
 
[e-mail]
 @dec.skip_without('numpy')
 def test_numpy_in_dict():
     import numpy
@@ -159,27 +152,25 b' def test_numpy_in_dict():'
             _scrub_nan(A)
             bufs = serialize_object(dict(a=A,b=1,c=range(20)))
             canned = pickle.loads(bufs[0])
-            yield nt.assert_true(canned['a'], CannedArray)
+            nt.assert_is_instance(canned['a'], CannedArray)
             d, r = unserialize_object(bufs)
             B = d['a']
-            yield nt.assert_equals(r, [])
-            yield nt.assert_equals(A.shape, B.shape)
-            yield nt.assert_equals(A.dtype, B.dtype)
-            yield assert_array_equal(A,B)
+            nt.assert_equal(r, [])
+            nt.assert_equal(A.shape, B.shape)
+            nt.assert_equal(A.dtype, B.dtype)
+            assert_array_equal(A,B)
 
[e-mail]
 def test_class():
     @interactive
     class C(object):
         a=5
     bufs = serialize_object(dict(C=C))
     canned = pickle.loads(bufs[0])
-    yield nt.assert_true(canned['C'], CannedClass)
+    nt.assert_is_instance(canned['C'], CannedClass)
     d, r = unserialize_object(bufs)
     C2 = d['C']
-    yield nt.assert_equal(C2.a, C.a)
+    nt.assert_equal(C2.a, C.a)
 
[e-mail]
 def test_class_oldstyle():
     @interactive
     class C:
@@ -187,42 +178,38 b' def test_class_oldstyle():'
     
     bufs = serialize_object(dict(C=C))
     canned = pickle.loads(bufs[0])
-    yield nt.assert_true(isinstance(canned['C'], CannedClass))
+    nt.assert_is_instance(canned['C'], CannedClass)
     d, r = unserialize_object(bufs)
     C2 = d['C']
-    yield nt.assert_equal(C2.a, C.a)
+    nt.assert_equal(C2.a, C.a)
 
[e-mail]
 def test_tuple():
     tup = (lambda x:x, 1)
     bufs = serialize_object(tup)
     canned = pickle.loads(bufs[0])
-    yield nt.assert_true(isinstance(canned, tuple))
+    nt.assert_is_instance(canned, tuple)
     t2, r = unserialize_object(bufs)
-    yield nt.assert_equal(t2[0](t2[1]), tup[0](tup[1]))
+    nt.assert_equal(t2[0](t2[1]), tup[0](tup[1]))
 
 point = namedtuple('point', 'x y')
 
[e-mail]
 def test_namedtuple():
     p = point(1,2)
     bufs = serialize_object(p)
     canned = pickle.loads(bufs[0])
-    yield nt.assert_true(isinstance(canned, point))
+    nt.assert_is_instance(canned, point)
     p2, r = unserialize_object(bufs, globals())
-    yield nt.assert_equal(p2.x, p.x)
-    yield nt.assert_equal(p2.y, p.y)
+    nt.assert_equal(p2.x, p.x)
+    nt.assert_equal(p2.y, p.y)
 
[e-mail]
 def test_list():
     lis = [lambda x:x, 1]
     bufs = serialize_object(lis)
     canned = pickle.loads(bufs[0])
-    yield nt.assert_true(isinstance(canned, list))
+    nt.assert_is_instance(canned, list)
     l2, r = unserialize_object(bufs)
-    yield nt.assert_equal(l2[0](l2[1]), lis[0](lis[1]))
+    nt.assert_equal(l2[0](l2[1]), lis[0](lis[1]))
 
[e-mail]
 def test_class_inheritance():
     @interactive
     class C(object):
@@ -234,8 +221,8 b' def test_class_inheritance():'
     
     bufs = serialize_object(dict(D=D))
     canned = pickle.loads(bufs[0])
-    yield nt.assert_true(canned['D'], CannedClass)
+    nt.assert_is_instance(canned['D'], CannedClass)
     d, r = unserialize_object(bufs)
     D2 = d['D']
-    yield nt.assert_equal(D2.a, D.a)
-    yield nt.assert_equal(D2.b, D.b)
+    nt.assert_equal(D2.a, D.a)
+    nt.assert_equal(D2.b, D.b)

IPython/kernel/zmq/tests/test_session.py

@@ -20,6 +20,12 b' from zmq.eventloop.zmqstream import ZMQStream'
 
 from IPython.kernel.zmq import session as ss
 
+def _bad_packer(obj):
+    raise TypeError("I don't work")
+
+def _bad_unpacker(bytes):
+    raise TypeError("I don't work either")
+
 class SessionTestCase(BaseZMQTestCase):
 
     def setUp(self):
@@ -222,4 +228,44 b' class TestSession(SessionTestCase):'
         self.assertTrue(len(session.digest_history) == 100)
         session._add_digest(uuid.uuid4().bytes)
         self.assertTrue(len(session.digest_history) == 91)
-
+    
+    def test_bad_pack(self):
+        try:
+            session = ss.Session(pack=_bad_packer)
+        except ValueError as e:
+            self.assertIn("could not serialize", str(e))
+            self.assertIn("don't work", str(e))
+        else:
+            self.fail("Should have raised ValueError")
+    
+    def test_bad_unpack(self):
+        try:
+            session = ss.Session(unpack=_bad_unpacker)
+        except ValueError as e:
+            self.assertIn("could not handle output", str(e))
+            self.assertIn("don't work either", str(e))
+        else:
+            self.fail("Should have raised ValueError")
+    
+    def test_bad_packer(self):
+        try:
+            session = ss.Session(packer=__name__ + '._bad_packer')
+        except ValueError as e:
+            self.assertIn("could not serialize", str(e))
+            self.assertIn("don't work", str(e))
+        else:
+            self.fail("Should have raised ValueError")
+    
+    def test_bad_unpacker(self):
+        try:
+            session = ss.Session(unpacker=__name__ + '._bad_unpacker')
+        except ValueError as e:
+            self.assertIn("could not handle output", str(e))
+            self.assertIn("don't work either", str(e))
+        else:
+            self.fail("Should have raised ValueError")
+    
+    def test_bad_roundtrip(self):
+        with self.assertRaises(ValueError):
+            session=  ss.Session(unpack=lambda b: 5)
+    

IPython/kernel/zmq/tests/test_start_kernel.py

@@ -8,10 +8,38 b' def test_ipython_start_kernel_userns():'
     cmd = ('from IPython import start_kernel\n'
            'ns = {"tre": 123}\n'
            'start_kernel(user_ns=ns)')
-    
+
     with setup_kernel(cmd) as client:
         msg_id = client.object_info('tre')
         msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
         content = msg['content']
         assert content['found']
-        nt.assert_equal(content['string_form'], u'123')
  No newline at end of file
+        nt.assert_equal(content['string_form'], u'123')
+
+        # user_module should be an instance of DummyMod
+        msg_id = client.execute("usermod = get_ipython().user_module")
+        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        content = msg['content']
+        nt.assert_equal(content['status'], u'ok')
+        msg_id = client.object_info('usermod')
+        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        content = msg['content']
+        assert content['found']
+        nt.assert_in('DummyMod', content['string_form'])
+
+def test_ipython_start_kernel_no_userns():
+    # Issue #4188 - user_ns should be passed to shell as None, not {}
+    cmd = ('from IPython import start_kernel\n'
+           'start_kernel()')
+
+    with setup_kernel(cmd) as client:
+        # user_module should not be an instance of DummyMod
+        msg_id = client.execute("usermod = get_ipython().user_module")
+        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        content = msg['content']
+        nt.assert_equal(content['status'], u'ok')
+        msg_id = client.object_info('usermod')
+        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        content = msg['content']
+        assert content['found']
+        nt.assert_not_in('DummyMod', content['string_form'])

IPython/kernel/zmq/zmqshell.py

@@ -86,14 +86,11 b' class ZMQDisplayPublisher(DisplayPublisher):'
             parent=self.parent_header, ident=self.topic,
         )
 
-    def clear_output(self, stdout=True, stderr=True, other=True):
-        content = dict(stdout=stdout, stderr=stderr, other=other)
-        
-        if stdout:
-            print('\r', file=sys.stdout, end='')
-        if stderr:
-            print('\r', file=sys.stderr, end='')
-        
+    def clear_output(self, wait=False):
+        content = dict(wait=wait)
+
+        print('\r', file=sys.stdout, end='')
+        print('\r', file=sys.stderr, end='')
         self._flush_streams()
         
         self.session.send(

IPython/lib/demo.py

@@ -14,14 +14,14 b' The classes are (see their docstrings for further details):'
  - Demo: pure python demos
 
  - IPythonDemo: demos with input to be processed by IPython as if it had been
- typed interactively (so magics work, as well as any other special syntax you
- may have added via input prefilters).
+   typed interactively (so magics work, as well as any other special syntax you
+   may have added via input prefilters).
 
  - LineDemo: single-line version of the Demo class.  These demos are executed
- one line at a time, and require no markup.
+   one line at a time, and require no markup.
 
  - IPythonLineDemo: IPython version of the LineDemo class (the demo is
- executed a line at a time, but processed via IPython).
+   executed a line at a time, but processed via IPython).
 
  - ClearMixin: mixin to make Demo classes with less visual clutter.  It
    declares an empty marquee and a pre_cmd that clears the screen before each
@@ -214,18 +214,18 b' class Demo(object):'
         Optional inputs:
 
           - title: a string to use as the demo name.  Of most use when the demo
-          you are making comes from an object that has no filename, or if you
-          want an alternate denotation distinct from the filename.
+            you are making comes from an object that has no filename, or if you
+            want an alternate denotation distinct from the filename.
 
           - arg_str(''): a string of arguments, internally converted to a list
-          just like sys.argv, so the demo script can see a similar
-          environment.
+            just like sys.argv, so the demo script can see a similar
+            environment.
 
           - auto_all(None): global flag to run all blocks automatically without
-          confirmation.  This attribute overrides the block-level tags and
-          applies to the whole demo.  It is an attribute of the object, and
-          can be changed at runtime simply by reassigning it to a boolean
-          value.
+            confirmation.  This attribute overrides the block-level tags and
+            applies to the whole demo.  It is an attribute of the object, and
+            can be changed at runtime simply by reassigning it to a boolean
+            value.
           """
         if hasattr(src, "read"):
              # It seems to be a file or a file-like object

IPython/lib/inputhook.py

@@ -481,6 +481,19 b' set_inputhook = inputhook_manager.set_inputhook'
 current_gui = inputhook_manager.current_gui
 clear_app_refs = inputhook_manager.clear_app_refs
 
+guis = {None: clear_inputhook,
+        GUI_NONE: clear_inputhook,
+        GUI_OSX: lambda app=False: None,
+        GUI_TK: enable_tk,
+        GUI_GTK: enable_gtk,
+        GUI_WX: enable_wx,
+        GUI_QT: enable_qt4, # qt3 not supported
+        GUI_QT4: enable_qt4,
+        GUI_GLUT: enable_glut,
+        GUI_PYGLET: enable_pyglet,
+        GUI_GTK3: enable_gtk3,
+}
+
 
 # Convenience function to switch amongst them
 def enable_gui(gui=None, app=None):
@@ -507,18 +520,6 b' def enable_gui(gui=None, app=None):'
     PyOS_InputHook wrapper object or the GUI toolkit app created, if there was
     one.
     """
-    guis = {None: clear_inputhook,
-            GUI_NONE: clear_inputhook,
-            GUI_OSX: lambda app=False: None,
-            GUI_TK: enable_tk,
-            GUI_GTK: enable_gtk,
-            GUI_WX: enable_wx,
-            GUI_QT: enable_qt4, # qt3 not supported
-            GUI_QT4: enable_qt4,
-            GUI_GLUT: enable_glut,
-            GUI_PYGLET: enable_pyglet,
-            GUI_GTK3: enable_gtk3,
-            }
     try:
         gui_hook = guis[gui]
     except KeyError:

IPython/lib/irunner.py

@@ -93,9 +93,9 b' class InteractiveRunner(object):'
           - program: command to execute the given program.
 
           - prompts: a list of patterns to match as valid prompts, in the
-          format used by pexpect.  This basically means that it can be either
-          a string (to be compiled as a regular expression) or a list of such
-          (it must be a true list, as pexpect does type checks).
+            format used by pexpect.  This basically means that it can be either
+            a string (to be compiled as a regular expression) or a list of such
+            (it must be a true list, as pexpect does type checks).
 
           If more than one prompt is given, the first is treated as the main
           program prompt and the others as 'continuation' prompts, like
@@ -107,19 +107,19 b' class InteractiveRunner(object):'
         Optional inputs:
 
           - args(None): optional list of strings to pass as arguments to the
-          child program.
+            child program.
 
           - out(sys.stdout): if given, an output stream to be used when writing
-          output.  The only requirement is that it must have a .write() method.
+            output.  The only requirement is that it must have a .write() method.
 
         Public members not parameterized in the constructor:
 
           - delaybeforesend(0): Newer versions of pexpect have a delay before
-          sending each new input.  For our purposes here, it's typically best
-          to just set this to zero, but if you encounter reliability problems
-          or want an interactive run to pause briefly at each prompt, just
-          increase this value (it is measured in seconds).  Note that this
-          variable is not honored at all by older versions of pexpect.
+            sending each new input.  For our purposes here, it's typically best
+            to just set this to zero, but if you encounter reliability problems
+            or want an interactive run to pause briefly at each prompt, just
+            increase this value (it is measured in seconds).  Note that this
+            variable is not honored at all by older versions of pexpect.
         """
 
         self.program = program
@@ -154,7 +154,7 b' class InteractiveRunner(object):'
 
         Inputs:
 
-          -fname: name of the file to execute.
+          - fname: name of the file to execute.
 
         See the run_source docstring for the meaning of the optional
         arguments."""
@@ -173,15 +173,15 b' class InteractiveRunner(object):'
         Inputs:
 
           - source: a string of code to be executed, or an open file object we
-          can iterate over.
+            can iterate over.
 
         Optional inputs:
 
           - interact(False): if true, start to interact with the running
-          program at the end of the script.  Otherwise, just exit.
+            program at the end of the script.  Otherwise, just exit.
 
           - get_output(False): if true, capture the output of the child process
-          (filtering the input commands out) and return it as a string.
+            (filtering the input commands out) and return it as a string.
 
         Returns:
           A string containing the process output, but only if requested.

IPython/lib/pretty.py

@@ -229,7 +229,17 b' class PrettyPrinter(_PrettyPrinterBase):'
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
-
+            
+    def break_(self):
+        """
+        Explicitly insert a newline into the output, maintaining correct indentation.
+        """
+        self.flush()
+        self.output.write(self.newline)
+        self.output.write(' ' * self.indentation)
+        self.output_width = self.indentation
+        self.buffer_width = 0
+        
 
     def begin_group(self, indent=0, open=''):
         """
@@ -478,8 +488,12 b' def _default_pprint(obj, p, cycle):'
     """
     klass = getattr(obj, '__class__', None) or type(obj)
     if getattr(klass, '__repr__', None) not in _baseclass_reprs:
-        # A user-provided repr.
-        p.text(repr(obj))
+        # A user-provided repr. Find newlines and replace them with p.break_()
+        output = repr(obj)
+        for idx,output_line in enumerate(output.splitlines()):
+            if idx:
+                p.break_()
+            p.text(output_line)
         return
     p.begin_group(1, '<')
     p.pretty(klass)

IPython/lib/tests/test_pretty.py

@@ -58,6 +58,23 b' class NoModule(object):'
 
 NoModule.__module__ = None
 
+class Breaking(object):
+    def _repr_pretty_(self, p, cycle):
+        with p.group(4,"TG: ",":"):
+            p.text("Breaking(")
+            p.break_()
+            p.text(")")
+
+class BreakingRepr(object):
+    def __repr__(self):
+        return "Breaking(\n)"
+
+class BreakingReprParent(object):
+    def _repr_pretty_(self, p, cycle):
+        with p.group(4,"TG: ",":"):
+            p.pretty(BreakingRepr())
+
+
 
 def test_indentation():
     """Test correct indentation in groups"""
@@ -118,3 +135,19 b' def test_pprint_nomod():'
     """
     output = pretty.pretty(NoModule)
     nt.assert_equal(output, 'NoModule')
+    
+def test_pprint_break():
+    """
+    Test that p.break_ produces expected output
+    """
+    output = pretty.pretty(Breaking())
+    expected = "TG: Breaking(\n    ):"
+    nt.assert_equal(output, expected)
+
+def test_pprint_break_repr():
+    """
+    Test that p.break_ is used in repr
+    """
+    output = pretty.pretty(BreakingReprParent())
+    expected = "TG: Breaking(\n    ):"
+    nt.assert_equal(output, expected)
  No newline at end of file

IPython/nbconvert/exporters/__init__.py

@@ -1,8 +1,9 b''
 from .export import *
 from .html import HTMLExporter
 from .slides import SlidesExporter
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 from .latex import LatexExporter
 from .markdown import MarkdownExporter
 from .python import PythonExporter
 from .rst import RSTExporter
+from .exporter import Exporter

IPython/nbconvert/exporters/export.py

@@ -19,6 +19,7 b' from IPython.nbformat.v3.nbbase import NotebookNode'
 from IPython.config import Config
 
 from .exporter import Exporter
+from .templateexporter import TemplateExporter
 from .html import HTMLExporter
 from .slides import SlidesExporter
 from .latex import LatexExporter
@@ -122,7 +123,7 b' def export(exporter, nb, **kw):'
     return output, resources
 
 exporter_map = dict(
-    custom=Exporter,
+    custom=TemplateExporter,
     html=HTMLExporter,
     slides=SlidesExporter,
     latex=LatexExporter,

IPython/nbconvert/exporters/exporter.py

@@ -19,56 +19,21 b' from __future__ import print_function, absolute_import'
 # Stdlib imports
 import io
 import os
-import inspect
 import copy
 import collections
 import datetime
 
-# other libs/dependencies
-from jinja2 import Environment, FileSystemLoader, ChoiceLoader, TemplateNotFound
 
 # IPython imports
 from IPython.config.configurable import LoggingConfigurable
 from IPython.config import Config
 from IPython.nbformat import current as nbformat
-from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict, Any
+from IPython.utils.traitlets import MetaHasTraits, Unicode, List
 from IPython.utils.importstring import import_item
-from IPython.utils.text import indent
-from IPython.utils import py3compat
+from IPython.utils import text, py3compat
 
 from IPython.nbconvert import preprocessors as nbpreprocessors
-from IPython.nbconvert import filters
 
-#-----------------------------------------------------------------------------
-# Globals and constants
-#-----------------------------------------------------------------------------
-
-#Jinja2 extensions to load.
-JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']
-
-default_filters = {
-        'indent': indent,
-        'markdown2html': filters.markdown2html,
-        'ansi2html': filters.ansi2html,
-        'filter_data_type': filters.DataTypeFilter,
-        'get_lines': filters.get_lines,
-        'highlight2html': filters.highlight2html,
-        'highlight2latex': filters.highlight2latex,
-        'ipython2python': filters.ipython2python,
-        'posix_path': filters.posix_path,
-        'markdown2latex': filters.markdown2latex,
-        'markdown2rst': filters.markdown2rst,
-        'comment_lines': filters.comment_lines,
-        'strip_ansi': filters.strip_ansi,
-        'strip_dollars': filters.strip_dollars,
-        'strip_files_prefix': filters.strip_files_prefix,
-        'html2text' : filters.html2text,
-        'add_anchor': filters.add_anchor,
-        'ansi2latex': filters.ansi2latex,
-        'strip_math_space': filters.strip_math_space,
-        'wrap_text': filters.wrap_text,
-        'escape_latex': filters.escape_latex,
-}
 
 #-----------------------------------------------------------------------------
 # Class
@@ -81,70 +46,21 b' class ResourcesDict(collections.defaultdict):'
 
 class Exporter(LoggingConfigurable):
     """
-    Exports notebooks into other file formats.  Uses Jinja 2 templating engine
-    to output new formats.  Inherit from this class if you are creating a new
-    template type along with new filters/preprocessors.  If the filters/
-    preprocessors provided by default suffice, there is no need to inherit from
-    this class.  Instead, override the template_file and file_extension
-    traits via a config file.
-
-    {filters}
+    Class containing methods that sequentially run a list of preprocessors on a 
+    NotebookNode object and then return the modified NotebookNode object and 
+    accompanying resources dict.
     """
-    
-    # finish the docstring
-    __doc__ = __doc__.format(filters = '- '+'\n    - '.join(default_filters.keys()))
-
-
-    template_file = Unicode(u'default',
-            config=True,
-            help="Name of the template file to use")
-    def _template_file_changed(self, name, old, new):
-        if new=='default':
-            self.template_file = self.default_template
-        else:
-            self.template_file = new
-        self.template = None
-        self._load_template()
-    
-    default_template = Unicode(u'')
-    template = Any()
-    environment = Any()
 
     file_extension = Unicode(
-        'txt', config=True, 
+        'txt', config=True,
         help="Extension of the file that should be written to disk"
         )
 
-    template_path = List(['.'], config=True)
-    def _template_path_changed(self, name, old, new):
-        self._load_template()
-
-    default_template_path = Unicode(
-        os.path.join("..", "templates"), 
-        help="Path where the template files are located.")
-
-    template_skeleton_path = Unicode(
-        os.path.join("..", "templates", "skeleton"), 
-        help="Path where the template skeleton files are located.") 
-
-    #Jinja block definitions
-    jinja_comment_block_start = Unicode("", config=True)
-    jinja_comment_block_end = Unicode("", config=True)
-    jinja_variable_block_start = Unicode("", config=True)
-    jinja_variable_block_end = Unicode("", config=True)
-    jinja_logic_block_start = Unicode("", config=True)
-    jinja_logic_block_end = Unicode("", config=True)
-    
-    #Extension that the template files use.    
-    template_extension = Unicode(".tpl", config=True)
-
     #Configurability, allows the user to easily add filters and preprocessors.
     preprocessors = List(config=True,
         help="""List of preprocessors, by name or namespace, to enable.""")
 
-    filters = Dict(config=True,
-        help="""Dictionary of filters, by name and namespace, to add to the Jinja
-        environment.""")
+    _preprocessors = None
 
     default_preprocessors = List([nbpreprocessors.coalesce_streams,
                                  nbpreprocessors.SVG2PDFPreprocessor,
@@ -152,42 +68,34 b' class Exporter(LoggingConfigurable):'
                                  nbpreprocessors.CSSHTMLHeaderPreprocessor,
                                  nbpreprocessors.RevealHelpPreprocessor,
                                  nbpreprocessors.LatexPreprocessor,
-                                 nbpreprocessors.SphinxPreprocessor],
+                                 nbpreprocessors.HighlightMagicsPreprocessor],
         config=True,
         help="""List of preprocessors available by default, by name, namespace, 
         instance, or type.""")
 
 
-    def __init__(self, config=None, extra_loaders=None, **kw):
+    def __init__(self, config=None, **kw):
         """
         Public constructor
-    
+
         Parameters
         ----------
         config : config
             User configuration instance.
-        extra_loaders : list[of Jinja Loaders]
-            ordered list of Jinja loader to find templates. Will be tried in order
-            before the default FileSystem ones.
-        template : str (optional, kw arg)
-            Template to use when exporting.
         """
         if not config:
             config = self.default_config
-        
+
         super(Exporter, self).__init__(config=config, **kw)
 
         #Init
-        self._init_template()
-        self._init_environment(extra_loaders=extra_loaders)
         self._init_preprocessors()
-        self._init_filters()
 
 
     @property
     def default_config(self):
         return Config()
-    
+
     def _config_changed(self, name, old, new):
         """When setting config, make sure to start with our default_config"""
         c = self.default_config
@@ -196,56 +104,18 b' class Exporter(LoggingConfigurable):'
         if c != old:
             self.config = c
         super(Exporter, self)._config_changed(name, old, c)
-        
-
-    def _load_template(self):
-        """Load the Jinja template object from the template file
-        
-        This is a no-op if the template attribute is already defined,
-        or the Jinja environment is not setup yet.
-        
-        This is triggered by various trait changes that would change the template.
-        """
-        if self.template is not None:
-            return
-        # called too early, do nothing
-        if self.environment is None:
-            return
-        # Try different template names during conversion.  First try to load the
-        # template by name with extension added, then try loading the template
-        # as if the name is explicitly specified, then try the name as a 
-        # 'flavor', and lastly just try to load the template by module name.
-        module_name = self.__module__.rsplit('.', 1)[-1]
-        try_names = []
-        if self.template_file:
-            try_names.extend([
-                self.template_file + self.template_extension,
-                self.template_file,
-                module_name + '_' + self.template_file + self.template_extension,
-            ])
-        try_names.append(module_name + self.template_extension)
-        for try_name in try_names:
-            self.log.debug("Attempting to load template %s", try_name)
-            try:
-                self.template = self.environment.get_template(try_name)
-            except (TemplateNotFound, IOError):
-                pass
-            except Exception as e:
-                self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True)
-            else:
-                self.log.info("Loaded template %s", try_name)
-                break
-    
+
+
     def from_notebook_node(self, nb, resources=None, **kw):
         """
         Convert a notebook from a notebook node instance.
-    
+
         Parameters
         ----------
         nb : Notebook node
-        resources : dict (**kw) 
-            of additional resources that can be accessed read/write by 
-            preprocessors and filters.
+        resources : dict (**kw)
+            of additional resources that can be accessed read/write by
+            preprocessors.
         """
         nb_copy = copy.deepcopy(nb)
         resources = self._init_resources(resources)
@@ -253,26 +123,20 b' class Exporter(LoggingConfigurable):'
         # Preprocess
         nb_copy, resources = self._preprocess(nb_copy, resources)
 
-        self._load_template()
-
-        if self.template is not None:
-            output = self.template.render(nb=nb_copy, resources=resources)
-        else:
-            raise IOError('template file "%s" could not be found' % self.template_file)
-        return output, resources
+        return nb_copy, resources
 
 
     def from_filename(self, filename, resources=None, **kw):
         """
         Convert a notebook from a notebook file.
-    
+
         Parameters
         ----------
         filename : str
             Full filename of the notebook file to open and convert.
         """
 
-        #Pull the metadata from the filesystem.
+        # Pull the metadata from the filesystem.
         if resources is None:
             resources = ResourcesDict()
         if not 'metadata' in resources or resources['metadata'] == '':
@@ -282,16 +146,16 b' class Exporter(LoggingConfigurable):'
         resources['metadata']['name'] = notebook_name
 
         modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
-        resources['metadata']['modified_date'] = modified_date.strftime("%B %d, %Y")
-        
+        resources['metadata']['modified_date'] = modified_date.strftime(text.date_format)
+
         with io.open(filename) as f:
-            return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw)
+            return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw)
 
 
     def from_file(self, file_stream, resources=None, **kw):
         """
         Convert a notebook from a notebook file.
-    
+
         Parameters
         ----------
         file_stream : file-like object
@@ -304,10 +168,10 b' class Exporter(LoggingConfigurable):'
         """
         Register a preprocessor.
         Preprocessors are classes that act upon the notebook before it is
-        passed into the Jinja templating engine.  Preprocessors are also
+        passed into the Jinja templating engine.  preprocessors are also
         capable of passing additional information to the Jinja
         templating engine.
-    
+
         Parameters
         ----------
         preprocessor : preprocessor
@@ -317,130 +181,43 b' class Exporter(LoggingConfigurable):'
         isclass = isinstance(preprocessor, type)
         constructed = not isclass
 
-        #Handle preprocessor's registration based on it's type
+        # Handle preprocessor's registration based on it's type
         if constructed and isinstance(preprocessor, py3compat.string_types):
-            #Preprocessor is a string, import the namespace and recursively call
-            #this register_preprocessor method
+            # Preprocessor is a string, import the namespace and recursively call
+            # this register_preprocessor method
             preprocessor_cls = import_item(preprocessor)
             return self.register_preprocessor(preprocessor_cls, enabled)
-        
+
         if constructed and hasattr(preprocessor, '__call__'):
-            #Preprocessor is a function, no need to construct it.
-            #Register and return the preprocessor.
+            # Preprocessor is a function, no need to construct it.
+            # Register and return the preprocessor.
             if enabled:
                 preprocessor.enabled = True
             self._preprocessors.append(preprocessor)
             return preprocessor
 
         elif isclass and isinstance(preprocessor, MetaHasTraits):
-            #Preprocessor is configurable.  Make sure to pass in new default for 
-            #the enabled flag if one was specified.
+            # Preprocessor is configurable.  Make sure to pass in new default for 
+            # the enabled flag if one was specified.
             self.register_preprocessor(preprocessor(parent=self), enabled)
 
         elif isclass:
-            #Preprocessor is not configurable, construct it
+            # Preprocessor is not configurable, construct it
             self.register_preprocessor(preprocessor(), enabled)
 
         else:
-            #Preprocessor is an instance of something without a __call__ 
-            #attribute.  
+            # Preprocessor is an instance of something without a __call__ 
+            # attribute.  
             raise TypeError('preprocessor')
 
 
-    def register_filter(self, name, jinja_filter):
-        """
-        Register a filter.
-        A filter is a function that accepts and acts on one string.  
-        The filters are accesible within the Jinja templating engine.
-    
-        Parameters
-        ----------
-        name : str
-            name to give the filter in the Jinja engine
-        filter : filter
-        """
-        if jinja_filter is None:
-            raise TypeError('filter')
-        isclass = isinstance(jinja_filter, type)
-        constructed = not isclass
-
-        #Handle filter's registration based on it's type
-        if constructed and isinstance(jinja_filter, py3compat.string_types):
-            #filter is a string, import the namespace and recursively call
-            #this register_filter method
-            filter_cls = import_item(jinja_filter)
-            return self.register_filter(name, filter_cls)
-        
-        if constructed and hasattr(jinja_filter, '__call__'):
-            #filter is a function, no need to construct it.
-            self.environment.filters[name] = jinja_filter
-            return jinja_filter
-
-        elif isclass and isinstance(jinja_filter, MetaHasTraits):
-            #filter is configurable.  Make sure to pass in new default for 
-            #the enabled flag if one was specified.
-            filter_instance = jinja_filter(parent=self)
-            self.register_filter(name, filter_instance )
-
-        elif isclass:
-            #filter is not configurable, construct it
-            filter_instance = jinja_filter()
-            self.register_filter(name, filter_instance)
-
-        else:
-            #filter is an instance of something without a __call__ 
-            #attribute.  
-            raise TypeError('filter')
-
-        
-    def _init_template(self):
-        """
-        Make sure a template name is specified.  If one isn't specified, try to
-        build one from the information we know.
-        """
-        self._template_file_changed('template_file', self.template_file, self.template_file)
-        
-
-    def _init_environment(self, extra_loaders=None):
-        """
-        Create the Jinja templating environment.
-        """
-        here = os.path.dirname(os.path.realpath(__file__))
-        loaders = []
-        if extra_loaders:
-            loaders.extend(extra_loaders)
-
-        paths = self.template_path
-        paths.extend([os.path.join(here, self.default_template_path),
-                      os.path.join(here, self.template_skeleton_path)])
-        loaders.append(FileSystemLoader(paths))
-
-        self.environment = Environment(
-            loader= ChoiceLoader(loaders),
-            extensions=JINJA_EXTENSIONS
-            )
-        
-        #Set special Jinja2 syntax that will not conflict with latex.
-        if self.jinja_logic_block_start:
-            self.environment.block_start_string = self.jinja_logic_block_start
-        if self.jinja_logic_block_end:
-            self.environment.block_end_string = self.jinja_logic_block_end
-        if self.jinja_variable_block_start:
-            self.environment.variable_start_string = self.jinja_variable_block_start
-        if self.jinja_variable_block_end:
-            self.environment.variable_end_string = self.jinja_variable_block_end
-        if self.jinja_comment_block_start:
-            self.environment.comment_start_string = self.jinja_comment_block_start
-        if self.jinja_comment_block_end:
-            self.environment.comment_end_string = self.jinja_comment_block_end
-
-    
     def _init_preprocessors(self):
         """
         Register all of the preprocessors needed for this exporter, disabled
         unless specified explicitly.
         """
-        self._preprocessors = []
+        if self._preprocessors is None:
+            self._preprocessors = []
 
         #Load default preprocessors (not necessarly enabled by default).
         if self.default_preprocessors:
@@ -451,21 +228,6 b' class Exporter(LoggingConfigurable):'
         if self.preprocessors:
             for preprocessor in self.preprocessors:
                 self.register_preprocessor(preprocessor, enabled=True)
- 
-
-    def _init_filters(self):
-        """
-        Register all of the filters required for the exporter.
-        """
-        
-        #Add default filters to the Jinja2 environment
-        for key, value in default_filters.items():
-            self.register_filter(key, value)
-
-        #Load user filters.  Overwrite existing filters if need be.
-        if self.filters:
-            for key, user_filter in self.filters.items():
-                self.register_filter(key, user_filter)
 
 
     def _init_resources(self, resources):
@@ -476,7 +238,7 b' class Exporter(LoggingConfigurable):'
         if not isinstance(resources, ResourcesDict):
             new_resources = ResourcesDict()
             new_resources.update(resources)
-            resources = new_resources 
+            resources = new_resources
 
         #Make sure the metadata extension exists in resources
         if 'metadata' in resources:
@@ -484,29 +246,28 b' class Exporter(LoggingConfigurable):'
                 resources['metadata'] = ResourcesDict(resources['metadata'])
         else:
             resources['metadata'] = ResourcesDict()
-            if not resources['metadata']['name']: 
+            if not resources['metadata']['name']:
                 resources['metadata']['name'] = 'Notebook'
 
         #Set the output extension
         resources['output_extension'] = self.file_extension
         return resources
-        
+
 
     def _preprocess(self, nb, resources):
         """
         Preprocess the notebook before passing it into the Jinja engine.
-        To preprocess the notebook is to apply all of the 
-    
+        To preprocess the notebook is to apply all of the
+
         Parameters
         ----------
         nb : notebook node
             notebook that is being exported.
         resources : a dict of additional resources that
             can be accessed read/write by preprocessors
-            and filters.
         """
-        
-        # Do a copy.deepcopy first, 
+
+        # Do a copy.deepcopy first,
         # we are never safe enough with what the preprocessors could do.
         nbc =  copy.deepcopy(nb)
         resc = copy.deepcopy(resources)

IPython/nbconvert/exporters/html.py

@@ -19,13 +19,13 b' from IPython.utils.traitlets import Unicode, List'
 from IPython.nbconvert import preprocessors
 from IPython.config import Config
 
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-class HTMLExporter(Exporter):
+class HTMLExporter(TemplateExporter):
     """
     Exports a basic HTML document.  This exporter assists with the export of
     HTML.  Inherit from it if you are writing your own HTML template and need
@@ -46,7 +46,10 b' class HTMLExporter(Exporter):'
         c = Config({
             'CSSHTMLHeaderPreprocessor':{
                 'enabled':True
-                }          
+                },
+            'HighlightMagicsPreprocessor': {
+                'enabled':True
+                }
             })
         c.merge(super(HTMLExporter,self).default_config)
         return c

IPython/nbconvert/exporters/latex.py

@@ -24,13 +24,13 b' from IPython.utils.traitlets import Unicode, List'
 from IPython.config import Config
 
 from IPython.nbconvert import filters, preprocessors
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-class LatexExporter(Exporter):
+class LatexExporter(TemplateExporter):
     """
     Exports to a Latex template.  Inherit from this class if your template is
     LaTeX based and you need custom tranformers/filters.  Inherit from it if 
@@ -45,7 +45,7 b' class LatexExporter(Exporter):'
         help="Extension of the file that should be written to disk")
 
     default_template = Unicode('article', config=True, help="""Template of the 
-        data format to use.  I.E. 'full' or 'basic'""")
+        data format to use.  I.E. 'article' or 'report'""")
 
     #Latex constants
     default_template_path = Unicode(
@@ -85,6 +85,9 b' class LatexExporter(Exporter):'
                  },
              'SphinxPreprocessor': {
                     'enabled':True
+                 },
+             'HighlightMagicsPreprocessor': {
+                    'enabled':True
                  }
          })
         c.merge(super(LatexExporter,self).default_config)

IPython/nbconvert/exporters/markdown.py

@@ -13,15 +13,16 b' Exporter that will export your ipynb to Markdown.'
 # Imports
 #-----------------------------------------------------------------------------
 
+from IPython.config import Config
 from IPython.utils.traitlets import Unicode
 
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-class MarkdownExporter(Exporter):
+class MarkdownExporter(TemplateExporter):
     """
     Exports to a markdown document (.md)
     """
@@ -29,3 +30,9 b' class MarkdownExporter(Exporter):'
     file_extension = Unicode(
         'md', config=True, 
         help="Extension of the file that should be written to disk")
+
+    @property
+    def default_config(self):
+        c = Config({'ExtractOutputPreprocessor':{'enabled':True}})
+        c.merge(super(MarkdownExporter,self).default_config)
+        return c

IPython/nbconvert/exporters/python.py

@@ -15,13 +15,13 b' Python exporter which exports Notebook code into a PY file.'
 
 from IPython.utils.traitlets import Unicode
 
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-class PythonExporter(Exporter):
+class PythonExporter(TemplateExporter):
     """
     Exports a Python code file.
     """

IPython/nbconvert/exporters/rst.py

@@ -16,13 +16,13 b' Exporter for exporting notebooks to restructured text.'
 from IPython.utils.traitlets import Unicode
 from IPython.config import Config
 
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-class RSTExporter(Exporter):
+class RSTExporter(TemplateExporter):
     """
     Exports restructured text documents.
     """

IPython/nbconvert/exporters/slides.py

@@ -19,13 +19,13 b' from IPython.utils.traitlets import Unicode'
 from IPython.nbconvert import preprocessors
 from IPython.config import Config
 
-from .exporter import Exporter
+from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-class SlidesExporter(Exporter):
+class SlidesExporter(TemplateExporter):
     """
     Exports slides
     """
@@ -47,6 +47,9 b' class SlidesExporter(Exporter):'
             'RevealHelpPreprocessor':{
                 'enabled':True,
                 },                
+            'HighlightMagicsPreprocessor': {
+                'enabled':True
+                }
             })
         c.merge(super(SlidesExporter,self).default_config)
         return c

IPython/nbconvert/exporters/tests/base.py

@@ -27,4 +27,4 b' class ExportersTestsBase(TestsBase):'
 
     def _get_notebook(self):
         return os.path.join(self._get_files_path(), 'notebook2.ipynb')
-                
  No newline at end of file
+                

IPython/nbconvert/exporters/tests/test_export.py

@@ -99,4 +99,4 b' class TestExport(ExportersTestsBase):'
             (output, resources) = export(None, self._get_notebook())
         except TypeError:
             pass
-                
  No newline at end of file
+                

IPython/nbconvert/exporters/tests/test_exporter.py

@@ -17,7 +17,7 b' Module with tests for exporter.py'
 from IPython.config import Config
 
 from .base import ExportersTestsBase
-from .cheese import CheesePreprocessor
+from ...preprocessors.base import Preprocessor
 from ..exporter import Exporter
 
 
@@ -25,84 +25,35 b' from ..exporter import Exporter'
 # Class
 #-----------------------------------------------------------------------------
 
+class PizzaPreprocessor(Preprocessor):
+    """Simple preprocessor that adds a 'pizza' entry to the NotebookNode.  Used 
+    to test Exporter.
+    """
+
+    def preprocess(self, nb, resources):
+        nb['pizza'] = 'cheese'
+        return nb, resources
+
+
 class TestExporter(ExportersTestsBase):
     """Contains test functions for exporter.py"""
 
 
     def test_constructor(self):
-        """
-        Can an Exporter be constructed?
-        """
+        """Can an Exporter be constructed?"""
         Exporter()
 
 
     def test_export(self):
-        """
-        Can an Exporter export something?
-        """
-        exporter = self._make_exporter()
-        (output, resources) = exporter.from_filename(self._get_notebook())
-        assert len(output) > 0
-
-
-    def test_extract_outputs(self):
-        """
-        If the ExtractOutputPreprocessor is enabled, are outputs extracted?
-        """
-        config = Config({'ExtractOutputPreprocessor': {'enabled': True}})
-        exporter = self._make_exporter(config=config)
-        (output, resources) = exporter.from_filename(self._get_notebook())
-        assert resources is not None
-        assert isinstance(resources['outputs'], dict)
-        assert len(resources['outputs']) > 0
-
-
-    def test_preprocessor_class(self):
-        """
-        Can a preprocessor be added to the preprocessors list by class type?
-        """
-        config = Config({'Exporter': {'preprocessors': [CheesePreprocessor]}})
-        exporter = self._make_exporter(config=config)
-        (output, resources) = exporter.from_filename(self._get_notebook())
-        assert resources is not None
-        assert resources['cheese'] == 'real'
-
-
-    def test_preprocessor_instance(self):
-        """
-        Can a preprocessor be added to the preprocessors list by instance?
-        """
-        config = Config({'Exporter': {'preprocessors': [CheesePreprocessor()]}})
-        exporter = self._make_exporter(config=config)
-        (output, resources) = exporter.from_filename(self._get_notebook())
-        assert resources is not None
-        assert resources['cheese'] == 'real'
-
-
-    def test_preprocessor_dottedobjectname(self):
-        """
-        Can a preprocessor be added to the preprocessors list by dotted object name?
-        """
-        config = Config({'Exporter': {'preprocessors': ['IPython.nbconvert.exporters.tests.cheese.CheesePreprocessor']}})
-        exporter = self._make_exporter(config=config)
-        (output, resources) = exporter.from_filename(self._get_notebook())
-        assert resources is not None
-        assert resources['cheese'] == 'real'
-
-
-    def test_preprocessor_via_method(self):
-        """
-        Can a preprocessor be added via the Exporter convenience method?
-        """
-        exporter = self._make_exporter()
-        exporter.register_preprocessor(CheesePreprocessor, enabled=True)
-        (output, resources) = exporter.from_filename(self._get_notebook())
-        assert resources is not None
-        assert resources['cheese'] == 'real'
-
-
-    def _make_exporter(self, config=None):
-        #Create the exporter instance, make sure to set a template name since
-        #the base Exporter doesn't have a template associated with it.
-        exporter = Exporter(config=config, template_file='python')
-        return exporter        
+        """Can an Exporter export something?"""
+        exporter = Exporter()
+        (notebook, resources) = exporter.from_filename(self._get_notebook())
+        assert isinstance(notebook, dict)
+
+
+    def test_preprocessor(self):
+        """Do preprocessors work?"""
+        config = Config({'Exporter': {'preprocessors': [PizzaPreprocessor()]}})
+        exporter = Exporter(config=config)
+        (notebook, resources) = exporter.from_filename(self._get_notebook())
+        self.assertEqual(notebook['pizza'], 'cheese')

IPython/nbconvert/exporters/tests/test_latex.py

@@ -44,18 +44,18 b' class TestLatexExporter(ExportersTestsBase):'
     @onlyif_cmds_exist('pandoc')
     def test_export_book(self):
         """
-        Can a LatexExporter export using 'book' template?
+        Can a LatexExporter export using 'report' template?
         """
-        (output, resources) = LatexExporter(template_file='book').from_filename(self._get_notebook())
+        (output, resources) = LatexExporter(template_file='report').from_filename(self._get_notebook())
         assert len(output) > 0
 
 
     @onlyif_cmds_exist('pandoc')
     def test_export_basic(self):
         """
-        Can a LatexExporter export using 'basic' template?
+        Can a LatexExporter export using 'article' template?
         """
-        (output, resources) = LatexExporter(template_file='basic').from_filename(self._get_notebook())
+        (output, resources) = LatexExporter(template_file='article').from_filename(self._get_notebook())
         assert len(output) > 0
 
 

IPython/nbconvert/filters/__init__.py

@@ -1,6 +1,7 b''
 from .ansi import *
+from .citation import *
 from .datatypefilter import *
 from .highlight import *
 from .latex import *
 from .markdown import *
-from .strings import *
  No newline at end of file
+from .strings import *

IPython/nbconvert/filters/highlight.py

@@ -14,7 +14,7 b' from within Jinja templates.'
 # Imports
 #-----------------------------------------------------------------------------
 
-from  pygments import highlight as pygements_highlight
+from pygments import highlight as pygements_highlight
 from pygments.lexers import get_lexer_by_name
 from pygments.formatters import HtmlFormatter
 from pygments.formatters import LatexFormatter
@@ -38,51 +38,73 b' __all__ = ['
 ]
 
 
-def highlight2html(source, language='ipython'):
+def highlight2html(source, language='ipython', metadata=None):
     """
     Return a syntax-highlighted version of the input source as html output.
-    
+
     Parameters
     ----------
     source : str
-        Source code to highlight the syntax of.
+        source of the cell to highlight
     language : str
-        Language to highlight the syntax of.
+        language to highlight the syntax of
+    metadata : NotebookNode cell metadata
+        metadata of the cell to highlight
     """
-    
-    return _pygment_highlight(source, HtmlFormatter(), language)
+
+    return _pygment_highlight(source, HtmlFormatter(), language, metadata)
 
 
-def highlight2latex(source, language='ipython'):
+def highlight2latex(source, language='ipython', metadata=None, strip_verbatim=False):
     """
     Return a syntax-highlighted version of the input source as latex output.
-    
+
     Parameters
     ----------
     source : str
-        Source code to highlight the syntax of.
+        source of the cell to highlight
     language : str
-        Language to highlight the syntax of.
+        language to highlight the syntax of
+    metadata : NotebookNode cell metadata
+        metadata of the cell to highlight
+    strip_verbatim : bool
+        remove the Verbatim environment that pygments provides by default
     """
-    return _pygment_highlight(source, LatexFormatter(), language)
+    latex = _pygment_highlight(source, LatexFormatter(), language, metadata)
+    if strip_verbatim:
+        latex = latex.replace(r'\begin{Verbatim}[commandchars=\\\{\}]' + '\n', '')
+        return latex.replace('\n\\end{Verbatim}\n', '')
+    else:
+        return latex
+
 
 
-def _pygment_highlight(source, output_formatter, language='ipython'):
+def _pygment_highlight(source, output_formatter, language='ipython', metadata=None):
     """
     Return a syntax-highlighted version of the input source
-    
+
     Parameters
     ----------
     source : str
-        Source code to highlight the syntax of.
+        source of the cell to highlight
     output_formatter : Pygments formatter
     language : str
-        Language to highlight the syntax of.
+        language to highlight the syntax of
+    metadata : NotebookNode cell metadata
+        metadata of the cell to highlight
     """
-    
+
+    # If the cell uses a magic extension language,
+    # use the magic language instead.
+    if language == 'ipython' \
+        and metadata \
+        and 'magics_language' in metadata:
+
+        language = metadata['magics_language']
+
     if language == 'ipython':
         lexer = IPythonLexer()
     else:
         lexer = get_lexer_by_name(language, stripall=True)
 
-    return pygements_highlight(source, lexer, output_formatter) 
+    return pygements_highlight(source, lexer, output_formatter)

IPython/nbconvert/filters/strings.py

@@ -19,6 +19,7 b' templates.'
 import os
 import re
 import textwrap
+from urllib2 import quote
 from xml.etree import ElementTree
 
 from IPython.core.interactiveshell import InteractiveShell
@@ -38,6 +39,8 b' __all__ = ['
     'get_lines',
     'ipython2python',
     'posix_path',
+    'path2url',
+    'add_prompts'
 ]
 
 
@@ -80,7 +83,7 b' def add_anchor(html):'
     
     For use in heading cells
     """
-    h = ElementTree.fromstring(py3compat.cast_bytes_py2(html))
+    h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding='utf-8'))
     link = html2text(h).replace(' ', '-')
     h.set('id', link)
     a = ElementTree.Element("a", {"class" : "anchor-link", "href" : "#" + link})
@@ -93,6 +96,16 b' def add_anchor(html):'
     return py3compat.decode(ElementTree.tostring(h), 'utf-8')
 
 
+def add_prompts(code, first='>>> ', cont='... '):
+    """Add prompts to code snippets"""
+    new_code = []
+    code_list = code.split('\n')
+    new_code.append(first + code_list[0])
+    for line in code_list[1:]:
+        new_code.append(cont + line)
+    return '\n'.join(new_code)
+
+    
 def strip_dollars(text):
     """
     Remove all dollar symbols from text
@@ -181,3 +194,8 b' def posix_path(path):'
     if os.path.sep != '/':
         return path.replace(os.path.sep, '/')
     return path
+
+def path2url(path):
+    """Turn a file path into a URL"""
+    parts = path.split(os.path.sep)
+    return '/'.join(quote(part) for part in parts)

IPython/nbconvert/filters/tests/test_ansi.py

@@ -39,7 +39,7 b' class TestAnsi(TestsBase):'
             'hello' : 'hello'}
 
         for inval, outval in correct_outputs.items():
-            yield self._try_strip_ansi(inval, outval)
+            self._try_strip_ansi(inval, outval)
 
 
     def _try_strip_ansi(self, inval, outval):
@@ -58,7 +58,7 b' class TestAnsi(TestsBase):'
             'hello' : 'hello'}
 
         for inval, outval in correct_outputs.items():
-            yield self._try_ansi2html(inval, outval)
+            self._try_ansi2html(inval, outval)
 
 
     def _try_ansi2html(self, inval, outval):
@@ -77,7 +77,7 b' class TestAnsi(TestsBase):'
             'hello' : 'hello'}
 
         for inval, outval in correct_outputs.items():
-            yield self._try_ansi2latex(inval, outval)
+            self._try_ansi2latex(inval, outval)
 
 
     def _try_ansi2latex(self, inval, outval):

IPython/nbconvert/filters/tests/test_highlight.py

@@ -49,13 +49,13 b' class TestHighlight(TestsBase):'
     def test_highlight2html(self):
         """highlight2html test"""
         for index, test in enumerate(self.tests):
-            yield self._try_highlight(highlight2html, test, self.tokens[index])
+            self._try_highlight(highlight2html, test, self.tokens[index])
 
 
     def test_highlight2latex(self):
         """highlight2latex test"""
         for index, test in enumerate(self.tests):
-            yield self._try_highlight(highlight2latex, test, self.tokens[index])
+            self._try_highlight(highlight2latex, test, self.tokens[index])
 
 
     def _try_highlight(self, method, test, tokens):

IPython/nbconvert/filters/tests/test_latex.py

@@ -35,7 +35,7 b' class TestLatex(TestsBase):'
             ('','')]
 
         for test in tests:
-            yield self._try_escape_latex(test[0], test[1])
+            self._try_escape_latex(test[0], test[1])
 
 
     def _try_escape_latex(self, test, result):
@@ -56,7 +56,7 b' class TestLatex(TestsBase):'
             ('','')]
 
         for test in tests:
-            yield self._try_strip_math_space(test[0], test[1])
+            self._try_strip_math_space(test[0], test[1])
 
 
     def _try_strip_math_space(self, test, result):

IPython/nbconvert/filters/tests/test_markdown.py

@@ -61,14 +61,14 b' class TestMarkdown(TestsBase):'
     def test_markdown2latex(self):
         """markdown2latex test"""
         for index, test in enumerate(self.tests):
-            yield self._try_markdown(markdown2latex, test, self.tokens[index])
+            self._try_markdown(markdown2latex, test, self.tokens[index])
 
 
     @dec.onlyif_cmds_exist('pandoc')
     def test_markdown2html(self):
         """markdown2html test"""
         for index, test in enumerate(self.tests):
-            yield self._try_markdown(markdown2html, test, self.tokens[index])
+            self._try_markdown(markdown2html, test, self.tokens[index])
 
 
     @dec.onlyif_cmds_exist('pandoc')
@@ -81,7 +81,7 b' class TestMarkdown(TestsBase):'
         tokens[1] = r'\*\*test'
 
         for index, test in enumerate(self.tests):
-            yield self._try_markdown(markdown2rst, test, tokens[index])
+            self._try_markdown(markdown2rst, test, tokens[index])
 
 
     def _try_markdown(self, method, test, tokens):

IPython/nbconvert/filters/tests/test_strings.py

@@ -15,10 +15,10 b' Module with tests for Strings'
 #-----------------------------------------------------------------------------
 import os
 
-from IPython.testing import decorators as dec
 from ...tests.base import TestsBase
 from ..strings import (wrap_text, html2text, add_anchor, strip_dollars, 
     strip_files_prefix, get_lines, comment_lines, ipython2python, posix_path,
+    add_prompts
 )
 
 
@@ -36,7 +36,7 b' class TestStrings(TestsBase):'
         As if the strings were thine, shouldst know of this.
         """
         for length in [30,5,1]:
-            yield self._confirm_wrap_text(test_text, length)
+            self._confirm_wrap_text(test_text, length)
     
 
     def _confirm_wrap_text(self, text, length):
@@ -73,7 +73,7 b' class TestStrings(TestsBase):'
             ('Hello', 'Hello'),
             ('W$o$rld', 'W$o$rld')]
         for test in tests:
-            yield self._try_strip_dollars(test[0], test[1])
+            self._try_strip_dollars(test[0], test[1])
 
 
     def _try_strip_dollars(self, test, result):
@@ -89,7 +89,7 b' class TestStrings(TestsBase):'
             ('My files are in `files/`', 'My files are in `files/`'),
             ('<a href="files/test.html">files/test.html</a>', '<a href="test.html">files/test.html</a>')]
         for test in tests:
-            yield self._try_files_prefix(test[0], test[1])
+            self._try_files_prefix(test[0], test[1])
 
 
     def _try_files_prefix(self, test, result):
@@ -121,8 +121,15 b' class TestStrings(TestsBase):'
             ignore_spaces=True, ignore_newlines=True)
     
     def test_posix_path(self):
+        """posix_path test"""
         path_list = ['foo', 'bar']
         expected = '/'.join(path_list)
         native = os.path.join(*path_list)
         filtered = posix_path(native)
         self.assertEqual(filtered, expected)
+    
+    def test_add_prompts(self):
+        """add_prompts test"""
+        text1 = """for i in range(10):\n  i += 1\n  print i"""
+        text2 = """>>> for i in range(10):\n...   i += 1\n...   print i"""
+        self.assertEqual(text2, add_prompts(text1))

IPython/nbconvert/nbconvertapp.py

@@ -25,6 +25,7 b' import glob'
 
 # From IPython
 from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
+from IPython.core.profiledir import ProfileDir
 from IPython.config import catch_config_error, Configurable
 from IPython.utils.traitlets import (
     Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum,
@@ -58,12 +59,11 b' nbconvert_aliases = {}'
 nbconvert_aliases.update(base_aliases)
 nbconvert_aliases.update({
     'to' : 'NbConvertApp.export_format',
-    'template' : 'Exporter.template_file',
+    'template' : 'TemplateExporter.template_file',
     'writer' : 'NbConvertApp.writer_class',
     'post': 'NbConvertApp.postprocessor_class',
     'output': 'NbConvertApp.output_base',
-    'offline-slides': 'RevealHelpTransformer.url_prefix',
-    'slide-notes': 'RevealHelpTransformer.speaker_notes'
+    'reveal-prefix': 'RevealHelpPreprocessor.url_prefix',
 })
 
 nbconvert_flags = {}
@@ -87,12 +87,13 b' class NbConvertApp(BaseIPythonApplication):'
         return logging.INFO
     
     def _classes_default(self):
-        classes = [NbConvertBase]
+        classes = [NbConvertBase, ProfileDir]
         for pkg in (exporters, preprocessors, writers):
             for name in dir(pkg):
                 cls = getattr(pkg, name)
                 if isinstance(cls, type) and issubclass(cls, Configurable):
                     classes.append(cls)
+        
         return classes
 
     description = Unicode(

IPython/nbconvert/postprocessors/__init__.py

@@ -1,3 +1,8 b''
 from .base import PostProcessorBase
 from .pdf import PDFPostProcessor
-from .serve import ServePostProcessor
+
+# protect against unavailable tornado
+try:
+    from .serve import ServePostProcessor
+except ImportError:
+    pass

IPython/nbconvert/postprocessors/pdf.py

@@ -26,35 +26,107 b' from .base import PostProcessorBase'
 class PDFPostProcessor(PostProcessorBase):
     """Writer designed to write to PDF files"""
 
-    iteration_count = Integer(3, config=True, help="""
+    latex_count = Integer(3, config=True, help="""
         How many times pdflatex will be called.
         """)
 
-    command = List(["pdflatex", "{filename}"], config=True, help="""
+    latex_command = List(["pdflatex", "{filename}"], config=True, help="""
         Shell command used to compile PDF.""")
 
+    bib_command = List(["bibtex", "{filename}"], config=True, help="""
+        Shell command used to run bibtex.""")
+
     verbose = Bool(False, config=True, help="""
         Whether or not to display the output of the compile call.
         """)
 
-    def postprocess(self, input):
-            """
-            Consume and write Jinja output a PDF.  
-            See files.py for more...
-            """        
-            command = [c.format(filename=input) for c in self.command]
-            self.log.info("Building PDF: %s", command)
-            with open(os.devnull, 'rb') as null:
-                stdout = subprocess.PIPE if not self.verbose else None
-                for index in range(self.iteration_count):
-                    p = subprocess.Popen(command, stdout=stdout, stdin=null)
-                    out, err = p.communicate()
-                    if p.returncode:
-                        if self.verbose:
-                            # verbose means I didn't capture stdout with PIPE,
-                            # so it's already been displayed and `out` is None.
-                            out = u''
-                        else:
-                            out = out.decode('utf-8', 'replace')
-                        self.log.critical(u"PDF conversion failed: %s\n%s", command, out)
-                        return
+    temp_file_exts = List(['.aux', '.bbl', '.blg', '.idx', '.log', '.out'], 
+        config=True, help="""
+        Filename extensions of temp files to remove after running.
+        """)
+
+    def run_command(self, command_list, filename, count, log_function):
+        """Run command_list count times.
+        
+        Parameters
+        ----------
+        command_list : list
+            A list of args to provide to Popen. Each element of this
+            list will be interpolated with the filename to convert.
+        filename : unicode
+            The name of the file to convert.
+        count : int
+            How many times to run the command.
+        
+        Returns
+        -------
+        continue : bool
+            A boolean indicating if the command was successful (True)
+            or failed (False).
+        """
+        command = [c.format(filename=filename) for c in command_list]
+        times = 'time' if count == 1 else 'times'
+        self.log.info("Running %s %i %s: %s", command_list[0], count, times, command)
+        with open(os.devnull, 'rb') as null:
+            stdout = subprocess.PIPE if not self.verbose else None
+            for index in range(count):
+                p = subprocess.Popen(command, stdout=stdout, stdin=null)
+                out, err = p.communicate()
+                if p.returncode:
+                    if self.verbose:
+                        # verbose means I didn't capture stdout with PIPE,
+                        # so it's already been displayed and `out` is None.
+                        out = u''
+                    else:
+                        out = out.decode('utf-8', 'replace')
+                    log_function(command, out)
+                    return False # failure
+        return True # success
+
+    def run_latex(self, filename):
+        """Run pdflatex self.latex_count times."""
+
+        def log_error(command, out):
+            self.log.critical(u"%s failed: %s\n%s", command[0], command, out)
+
+        return self.run_command(self.latex_command, filename, 
+            self.latex_count, log_error)
+
+    def run_bib(self, filename):
+        """Run bibtex self.latex_count times."""
+        filename = os.path.splitext(filename)[0]
+
+        def log_error(command, out):
+            self.log.warn('%s had problems, most likely because there were no citations',
+                command[0])
+            self.log.debug(u"%s output: %s\n%s", command[0], command, out)
+
+        return self.run_command(self.bib_command, filename, 1, log_error)
+
+    def clean_temp_files(self, filename):
+        """Remove temporary files created by pdflatex/bibtext."""
+        self.log.info("Removing temporary LaTeX files")
+        filename = os.path.splitext(filename)[0]
+        for ext in self.temp_file_exts:
+            try:
+                os.remove(filename+ext)
+            except OSError:
+                pass
+
+    def postprocess(self, filename):
+        """Build a PDF by running pdflatex and bibtex"""
+        self.log.info("Building PDF")
+        cont = self.run_latex(filename)
+        if cont:
+            cont = self.run_bib(filename)
+        else:
+            self.clean_temp_files(filename)
+            return
+        if cont:
+            cont = self.run_latex(filename)
+        self.clean_temp_files(filename)
+        filename = os.path.splitext(filename)[0]
+        if os.path.isfile(filename+'.pdf'):
+            self.log.info('PDF successfully created')
+        return
+

IPython/nbconvert/postprocessors/serve.py

@@ -1,6 +1,4 b''
-"""
-Contains postprocessor for serving nbconvert output.
-"""
+"""PostProcessor for serving reveal.js HTML slideshows."""
 #-----------------------------------------------------------------------------
 #Copyright (c) 2013, the IPython Development Team.
 #
@@ -16,40 +14,94 b' Contains postprocessor for serving nbconvert output.'
 import os
 import webbrowser
 
-from BaseHTTPServer import HTTPServer
-from SimpleHTTPServer import SimpleHTTPRequestHandler
+from tornado import web, ioloop, httpserver
+from tornado.httpclient import AsyncHTTPClient
 
-from IPython.utils.traitlets import Bool
+from IPython.utils.traitlets import Bool, Unicode, Int
 
 from .base import PostProcessorBase
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
+
+class ProxyHandler(web.RequestHandler):
+    """handler the proxies requests from a local prefix to a CDN"""
+    @web.asynchronous
+    def get(self, prefix, url):
+        """proxy a request to a CDN"""
+        proxy_url = "/".join([self.settings['cdn'], url])
+        client = self.settings['client']
+        client.fetch(proxy_url, callback=self.finish_get)
+    
+    def finish_get(self, response):
+        """finish the request"""
+        # copy potentially relevant headers
+        for header in ["Content-Type", "Cache-Control", "Date", "Last-Modified", "Expires"]:
+            if header in response.headers:
+                self.set_header(header, response.headers[header])
+        self.finish(response.body)
+
 class ServePostProcessor(PostProcessorBase):
-    """Post processor designed to serve files"""
+    """Post processor designed to serve files
+    
+    Proxies reveal.js requests to a CDN if no local reveal.js is present
+    """
 
 
     open_in_browser = Bool(True, config=True,
-                           help="""Set to False to deactivate
-                           the opening of the browser""")
+        help="""Should the browser be opened automatically?"""
+    )
+    reveal_cdn = Unicode("https://cdn.jsdelivr.net/reveal.js/2.4.0", config=True,
+        help="""URL for reveal.js CDN."""
+    )
+    reveal_prefix = Unicode("reveal.js", config=True, help="URL prefix for reveal.js")
+    ip = Unicode("127.0.0.1", config=True, help="The IP address to listen on.")
+    port = Int(8000, config=True, help="port for the server to listen on.")
 
     def postprocess(self, input):
-        """
-        Simple implementation to serve the build directory.
-        """
-
+        """Serve the build directory with a webserver."""
+        dirname, filename = os.path.split(input)
+        handlers = [
+            (r"/(.+)", web.StaticFileHandler, {'path' : dirname}),
+            (r"/", web.RedirectHandler, {"url": "/%s" % filename})
+        ]
+        
+        if ('://' in self.reveal_prefix or self.reveal_prefix.startswith("//")):
+            # reveal specifically from CDN, nothing to do
+            pass
+        elif os.path.isdir(os.path.join(dirname, self.reveal_prefix)):
+            # reveal prefix exists
+            self.log.info("Serving local %s", self.reveal_prefix)
+        else:
+            self.log.info("Redirecting %s requests to %s", self.reveal_prefix, self.reveal_cdn)
+            handlers.insert(0, (r"/(%s)/(.*)" % self.reveal_prefix, ProxyHandler))
+        
+        app = web.Application(handlers,
+            cdn=self.reveal_cdn,
+            client=AsyncHTTPClient(),
+        )
+        # hook up tornado logging to our logger
+        from tornado import log
+        log.app_log = self.log
+    
+        http_server = httpserver.HTTPServer(app)
+        http_server.listen(self.port, address=self.ip)
+        url = "http://%s:%i/%s" % (self.ip, self.port, filename)
+        print("Serving your slides at %s" % url)
+        print("Use Control-C to stop this server")
+        if self.open_in_browser:
+            webbrowser.open(url, new=2)
         try:
-            dirname, filename = os.path.split(input)
-            if dirname:
-                os.chdir(dirname)
-            httpd = HTTPServer(('127.0.0.1', 8000), SimpleHTTPRequestHandler)
-            sa = httpd.socket.getsockname()
-            url = "http://" + sa[0] + ":" + str(sa[1]) + "/" + filename
-            if self.open_in_browser:
-                webbrowser.open(url, new=2)
-            print("Serving your slides on " + url)
-            print("Use Control-C to stop this server.")
-            httpd.serve_forever()
+            ioloop.IOLoop.instance().start()
         except KeyboardInterrupt:
-            print("The server is shut down.")
+            print("\nInterrupted")
+
+def main(path):
+    """allow running this module to serve the slides"""
+    server = ServePostProcessor()
+    server(path)
+    
+if __name__ == '__main__':
+    import sys
+    main(sys.argv[1])

IPython/nbconvert/postprocessors/tests/test_pdf.py

@@ -62,3 +62,7 b' class TestPDF(TestsBase):'
 
             # Check that the PDF was created.
             assert os.path.isfile('a.pdf')
+            
+            # Make sure that temp files are cleaned up
+            for ext in processor.temp_file_exts:
+                assert not os.path.isfile('a'+ext)

IPython/nbconvert/preprocessors/__init__.py

@@ -5,8 +5,8 b' from .svg2pdf import SVG2PDFPreprocessor'
 from .extractoutput import ExtractOutputPreprocessor
 from .revealhelp import RevealHelpPreprocessor
 from .latex import LatexPreprocessor
-from .sphinx import SphinxPreprocessor
 from .csshtmlheader import CSSHTMLHeaderPreprocessor
+from .highlightmagics import HighlightMagicsPreprocessor
 
 # decorated function Preprocessors
 from .coalescestreams import coalesce_streams

IPython/nbconvert/preprocessors/coalescestreams.py

@@ -19,7 +19,8 b' def cell_preprocessor(function):'
     Wrap a function to be executed on all cells of a notebook
     
     Wrapped Parameters
-    ----------
+    ------------------
+    
     cell : NotebookNode cell
         Notebook cell being processed
     resources : dictionary
@@ -69,7 +70,7 b' def coalesce_streams(cell, resources, index):'
             last.text += output.text
         else:
             new_outputs.append(output)
-        last = output
+            last = output
 
     cell.outputs = new_outputs
     return cell, resources

IPython/nbconvert/preprocessors/latex.py

@@ -14,9 +14,12 b' they are converted.'
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
+import os
 
-# Our own imports
-# Needed to override preprocessor
+# Third-party import, needed for Pygments latex definitions.
+from pygments.formatters import LatexFormatter
+
+# ipy imports
 from .base import (Preprocessor)
 from IPython.nbconvert import filters
 
@@ -29,6 +32,24 b' class LatexPreprocessor(Preprocessor):'
     Converter for latex destined documents.
     """
 
+    def preprocess(self, nb, resources):
+        """
+        Preprocessing to apply on each notebook.
+        
+        Parameters
+        ----------
+        nb : NotebookNode
+            Notebook being converted
+        resources : dictionary
+            Additional resources used in the conversion process.  Allows
+            preprocessors to pass variables into the Jinja engine.
+        """
+        # Generate Pygments definitions for Latex 
+        resources["latex"] = {}
+        resources["latex"]["pygments_definitions"] = LatexFormatter().get_style_defs()
+        return super(LatexPreprocessor, self).preprocess(nb, resources)
+
+
     def preprocess_cell(self, cell, resources, index):
         """
         Apply a transformation on each cell,

IPython/nbconvert/preprocessors/revealhelp.py

@@ -24,15 +24,14 b' from IPython.utils.traitlets import Unicode, Bool'
 
 class RevealHelpPreprocessor(Preprocessor):
 
-    url_prefix = Unicode('//cdn.jsdelivr.net/reveal.js/2.4.0',
-                         config=True,
-                         help="""If you want to use a local reveal.js library,
-                         use 'url_prefix':'reveal.js' in your config object.""")
-
-    speaker_notes = Bool(False, 
-                         config=True, 
-                         help="""If you want to use the speaker notes 
-                         set this to True.""")
+    url_prefix = Unicode('reveal.js', config=True,
+                         help="""The URL prefix for reveal.js.
+                         This can be a a relative URL for a local copy of reveal.js,
+                         or point to a CDN.
+                         
+                         For speaker notes to work, a local reveal.js prefix must be used.
+                         """
+    )
 
     def preprocess(self, nb, resources):
         """
@@ -65,30 +64,4 b' class RevealHelpPreprocessor(Preprocessor):'
         if not isinstance(resources['reveal'], dict):
             resources['reveal'] = {}
         resources['reveal']['url_prefix'] = self.url_prefix
-        resources['reveal']['notes_prefix'] = self.url_prefix
-
-        cdn = 'http://cdn.jsdelivr.net/reveal.js/2.4.0'
-        local = 'local'
-        html_path = 'plugin/notes/notes.html'
-        js_path = 'plugin/notes/notes.js'
-
-        html_infile = os.path.join(cdn, html_path)
-        js_infile = os.path.join(cdn, js_path)
-        html_outfile = os.path.join(local, html_path)
-        js_outfile = os.path.join(local, js_path)
-
-        if self.speaker_notes:
-            if 'outputs' not in resources:
-                resources['outputs'] = {}
-            resources['outputs'][html_outfile] = self.notes_helper(html_infile)
-            resources['outputs'][js_outfile] = self.notes_helper(js_infile)
-            resources['reveal']['notes_prefix'] = local
-
         return nb, resources
-
-    def notes_helper(self, infile):
-        """Helper function to get the content from an url."""
-
-        content = urllib2.urlopen(infile).read()
-
-        return content

IPython/nbconvert/preprocessors/tests/base.py

@@ -36,7 +36,7 b' class PreprocessorTestsBase(TestsBase):'
                    nbformat.new_output(output_type="stream", stream="stdout", output_text="d"),
                    nbformat.new_output(output_type="stream", stream="stderr", output_text="e"),
                    nbformat.new_output(output_type="stream", stream="stderr", output_text="f"),
-                   nbformat.new_output(output_type="png", output_png=b'Zw==')] #g
+                   nbformat.new_output(output_type="png", output_png='Zw==')] #g
         
         cells=[nbformat.new_code_cell(input="$ e $", prompt_number=1,outputs=outputs),
                nbformat.new_text_cell('markdown', source="$ e $")]

IPython/nbconvert/preprocessors/tests/test_coalescestreams.py

@@ -14,6 +14,8 b' Module with tests for the coalescestreams preprocessor'
 # Imports
 #-----------------------------------------------------------------------------
 
+from IPython.nbformat import current as nbformat
+
 from .base import PreprocessorTestsBase
 from ..coalescestreams import coalesce_streams
 
@@ -35,4 +37,24 b' class TestCoalesceStreams(PreprocessorTestsBase):'
         self.assertEqual(outputs[1].output_type, "text")
         self.assertEqual(outputs[2].text, "cd")
         self.assertEqual(outputs[3].text, "ef")
-    
+
+
+    def test_coalesce_sequenced_streams(self):
+        """Can the coalesce streams preprocessor merge a sequence of streams?"""
+
+        outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="1"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="2"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="3"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="4"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="5"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="6"),
+                   nbformat.new_output(output_type="stream", stream="stdout", output_text="7")]
+        cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)]
+        worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)]
+
+        nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets)
+        res = self.build_resources()
+        nb, res = coalesce_streams(nb, res)
+        outputs = nb.worksheets[0].cells[0].outputs
+        self.assertEqual(outputs[0].text, u'01234567')

IPython/nbconvert/preprocessors/tests/test_extractoutput.py

@@ -46,12 +46,14 b' class TestExtractOutput(PreprocessorTestsBase):'
         nb, res = preprocessor(nb, res)
 
         # Check if text was extracted.
-        assert 'text_filename' in nb.worksheets[0].cells[0].outputs[1]
-        text_filename = nb.worksheets[0].cells[0].outputs[1]['text_filename']
+        output = nb.worksheets[0].cells[0].outputs[1]
+        assert 'text_filename' in output
+        text_filename = output['text_filename']
 
         # Check if png was extracted.
-        assert 'png_filename' in nb.worksheets[0].cells[0].outputs[6]
-        png_filename = nb.worksheets[0].cells[0].outputs[6]['png_filename']
+        output = nb.worksheets[0].cells[0].outputs[6]
+        assert 'png_filename' in output
+        png_filename = output['png_filename']
 
         # Verify text output
         assert text_filename in res['outputs']

IPython/nbconvert/preprocessors/tests/test_highlightmagics.py

@@ -1,5 +1,5 b''
 """
-Module with tests for the sphinx preprocessor
+Module with tests for the HighlightMagics preprocessor
 """
 
 #-----------------------------------------------------------------------------
@@ -15,43 +15,54 b' Module with tests for the sphinx preprocessor'
 #-----------------------------------------------------------------------------
 
 from .base import PreprocessorTestsBase
-from ..sphinx import SphinxPreprocessor
+from ..highlightmagics import HighlightMagicsPreprocessor
 
 
 #-----------------------------------------------------------------------------
 # Class
 #-----------------------------------------------------------------------------
 
-class TestSphinx(PreprocessorTestsBase):
-    """Contains test functions for sphinx.py"""
+class TestHighlightMagics(PreprocessorTestsBase):
+    """Contains test functions for highlightmagics.py"""
 
 
     def build_preprocessor(self):
         """Make an instance of a preprocessor"""
-        preprocessor = SphinxPreprocessor()
+        preprocessor = HighlightMagicsPreprocessor()
         preprocessor.enabled = True
         return preprocessor
 
-
     def test_constructor(self):
-        """Can a SphinxPreprocessor be constructed?"""
+        """Can a HighlightMagicsPreprocessor be constructed?"""
         self.build_preprocessor()
-    
 
-    def test_resources(self):
-        """Make sure the SphinxPreprocessor adds the appropriate resources to the
-        resources dict."""
+    def test_tagging(self):
+        """Test the HighlightMagicsPreprocessor tagging"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """%%R -i x,y -o XYcoef
+            lm.fit <- lm(y~x)
+            par(mfrow=c(2,2))
+            print(summary(lm.fit))
+            plot(lm.fit)
+            XYcoef <- coef(lm.fit)"""
+
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' in nb.worksheets[0].cells[0]['metadata'])
+
+        self.assertEqual(nb.worksheets[0].cells[0]['metadata']['magics_language'], 'r')
+
+    def test_no_false_positive(self):
+        """Test that HighlightMagicsPreprocessor does not tag false positives"""
         nb = self.build_notebook()
         res = self.build_resources()
         preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """# this should not be detected
+                print(\"""
+                %%R -i x, y
+                \""")"""
         nb, res = preprocessor(nb, res)
-        assert "author" in res['sphinx']
-        assert "version" in res['sphinx']
-        assert "release" in res['sphinx']
-        assert "date" in res['sphinx']
-        assert "chapterstyle" in res['sphinx']
-        assert "outputstyle" in res['sphinx']
-        assert "centeroutput" in res['sphinx']
-        assert "header" in res['sphinx']
-        assert "texinputs" in res['sphinx']
-        assert "pygment_definitions" in res['sphinx']
+
+        assert('magics_language' not in nb.worksheets[0].cells[0]['metadata'])
  No newline at end of file

IPython/nbconvert/templates/html_basic.tpl

@@ -36,7 +36,7 b' In [{{ cell.prompt_number }}]:'
 
 {% block input %}
 <div class="input_area box-flex1">
-{{ cell.input | highlight2html }}
+{{ cell.input | highlight2html(metadata=cell.metadata) }}
 </div>
 {%- endblock input %}
 
@@ -135,6 +135,12 b' unknown type {{ cell.type }}'
 </pre>
 {%- endblock -%}
 
+{%- block data_javascript %}
+<script type="text/javascript">
+{{ output.javascript }}
+</script>
+{%- endblock -%}
+
 {%- block display_data scoped -%}
 <div class="box-flex1 output_subarea output_display_data">
 {{ super() }}

IPython/nbconvert/templates/latex/latex_article.tplx

@@ -1,26 +1,17 b''
-((============================================================================
-  NBConvert Sphinx-Latex HowTo Template
 
-  Purpose: Allow export of PDF friendly Latex inspired by Sphinx HowTo 
-           document style.  Most of the is derived directly from Sphinx source.
+% Default to the notebook output style
+((* if not cell_style is defined *))
+    ((* set cell_style = 'style_ipython.tplx' *))
+((* endif *))
 
-  Inheritance: null>display_priority>sphinx
+% Inherit from the specified cell style.
+((* extends cell_style *))
 
-  ==========================================================================))
 
-((*- extends 'sphinx.tplx' -*))
+%===============================================================================
+% Latex Article
+%===============================================================================
 
-
-((* set parentdocumentclass = 'article' *))
-((* set documentclass = 'howto' *))
-
-((* block h1 -*))part((* endblock h1 -*))
-((* block h2 -*))section((* endblock h2 -*))
-((* block h3 -*))subsection((* endblock h3 -*))
-((* block h4 -*))subsubsection((* endblock h4 -*))
-((* block h5 -*))paragraph((* endblock h5 -*))
-((* block h6 -*))subparagraph((* endblock h6 -*))
-
-% Diasble table of contents for howto
-((* block toc *))
-((* endblock toc *))
+((* block docclass *))
+\documentclass{article}
+((* endblock docclass *))
  No newline at end of file

IPython/nbconvert/templates/latex/skeleton/display_priority.tplx

@@ -1,4 +1,7 b''
-((= autogenerated file do not edit =))
+((= Auto-generated template file, DO NOT edit directly!
+    To edit this file, please refer to ../../skeleton/README.md =))
+
+
 ((*- extends 'null.tplx' -*))
 
 ((=display data priority=))
@@ -30,10 +33,13 b''
             ((*- block data_text -*))
             ((*- endblock -*))
         ((*- endif -*))
-
         ((*- if type in ['latex']*))
             ((*- block data_latex -*))
             ((*- endblock -*))
         ((*- endif -*))
+        ((*- if type in ['javascript']*))
+            ((*- block data_javascript -*))
+            ((*- endblock -*))
+        ((*- endif -*))
     ((*- endfor -*))
 ((*- endblock data_priority -*))

IPython/nbconvert/templates/latex/skeleton/null.tplx

@@ -1,11 +1,14 b''
-((= autogenerated file do not edit =))
+((= Auto-generated template file, DO NOT edit directly!
+    To edit this file, please refer to ../../skeleton/README.md =))
+
+
 ((=
 
-DO NOT USE THIS AS A BASE WORK,
+DO NOT USE THIS AS A BASE,
 IF YOU ARE COPY AND PASTING THIS FILE
-YOU ARE PROBABLY DOING THINGS WRONG.
+YOU ARE PROBABLY DOING THINGS INCORRECTLY.
 
-Null template, Does nothing except defining a basic structure
+Null template, does nothing except defining a basic structure
 To layout the different blocks of a notebook.
 
 Subtemplates can override blocks to define their custom representation.

IPython/nbconvert/templates/markdown.tpl

@@ -2,19 +2,13 b''
 
 
 {% block in_prompt %}
-In[{{ cell.prompt_number if cell.prompt_number else ' ' }}]:
 {% endblock in_prompt %}
 
 {% block output_prompt %}
-{% if cell.haspyout %}
-Out[{{ cell.prompt_number }}]:
-{%- endif %}
 {%- endblock output_prompt %}
 
 {% block input %}
-```
-{{ cell.input }}
-```
+{{ cell.input | indent(4)}}
 {% endblock input %}
 
 {% block pyerr %}
@@ -26,6 +20,7 b' Out[{{ cell.prompt_number }}]:'
 {% endblock traceback_line %}
 
 {% block pyout %}
+
 {% block data_priority scoped %}
 {{ super() }}
 {% endblock %}
@@ -36,23 +31,25 b' Out[{{ cell.prompt_number }}]:'
 {% endblock stream %}
 
 {% block data_svg %}
-[!image]({{ output.svg_filename }})
+![svg]({{ output.svg_filename | path2url }})
 {% endblock data_svg %}
 
 {% block data_png %}
-[!image]({{ output.png_filename }})
+![png]({{ output.png_filename | path2url }})
 {% endblock data_png %}
 
 {% block data_jpg %}
-[!image]({{ output.jpg_filename }})
+![jpeg]({{ output.jpeg_filename | path2url }})
 {% endblock data_jpg %}
 
 {% block data_latex %}
-$$
 {{ output.latex }}
-$$
 {% endblock data_latex %}
 
+{% block data_html scoped %}
+{{ output.html }}
+{% endblock data_html %}
+
 {% block data_text scoped %}
 {{ output.text | indent }}
 {% endblock data_text %}
@@ -61,6 +58,7 b' $$'
 {{ cell.source | wrap_text(80) }}
 {% endblock markdowncell %}
 
+
 {% block headingcell scoped %}
 {{ '#' * cell.level }} {{ cell.source | replace('\n', ' ') }}
 {% endblock headingcell %}

IPython/nbconvert/templates/rst.tpl

@@ -2,25 +2,22 b''
 
 
 {% block in_prompt %}
-
-In[{{ cell.prompt_number if cell.prompt_number else ' ' }}]:
-
-.. code:: python
-
 {% endblock in_prompt %}
 
 {% block output_prompt %}
-{% if cell.haspyout -%}
-    Out[{{ cell.prompt_number }}]:
-{% endif %}
 {% endblock output_prompt %}
 
 {% block input %}
+{%- if not cell.input.isspace() -%}
+.. code:: python
+
 {{ cell.input | indent}}
+{%- endif -%}
 {% endblock input %}
 
 {% block pyerr %}
 ::
+
 {{ super() }}
 {% endblock pyerr %}
 
@@ -49,19 +46,27 b" In[{{ cell.prompt_number if cell.prompt_number else ' ' }}]:"
 {% endblock data_png %}
 
 {% block data_jpg %}
-..jpg image:: {{ output.jpg_filename }}
+.. image:: {{ output.jpeg_filename }}
 {% endblock data_jpg %}
 
 {% block data_latex %}
 .. math::
-{{ output.latex | indent }}
+
+{{ output.latex | strip_dollars | indent }}
 {% endblock data_latex %}
 
 {% block data_text scoped %}
 .. parsed-literal::
+
 {{ output.text | indent }}
 {% endblock data_text %}
 
+{% block data_html scoped %}
+.. raw:: html
+
+{{ output.html | indent }}
+{% endblock data_html %}
+
 {% block markdowncell scoped %}
 {{ cell.source | markdown2rst }}
 {% endblock markdowncell %}

IPython/nbconvert/templates/skeleton/Makefile

@@ -1,12 +1,14 b''
+TPLS := $(patsubst %.tpl,../latex/skeleton/%.tplx,$(wildcard *.tpl))
 
+all: clean $(TPLS)
 
-all: tex/null.tplx tex/display_priority.tplx
-
-# convert jinja syntax to tex
-# cf http://flask.pocoo.org/snippets/55/
-tex/%.tplx: %.tpl
+# Convert standard Jinja2 syntax to LaTeX safe Jinja2
+# see http://flask.pocoo.org/snippets/55/ for more info
+../latex/skeleton/%.tplx: %.tpl
 	@echo 'generating tex equivalent of $^: $@'
-	@echo '((= autogenerated file do not edit =))' > $@
+	@echo '((= Auto-generated template file, DO NOT edit directly!\n' \
+		  '   To edit this file, please refer to ../../skeleton/README.md' \
+		  '=))\n\n' > $@
 	@sed \
 		-e 's/{%/((*/g' \
 		-e 's/%}/*))/g' \
@@ -17,7 +19,6 b' tex/%.tplx: %.tpl'
 		-e "s/tpl'/tplx'/g" \
 		$^ >> $@
 
-
 clean:
 	@echo "cleaning generated tplx files..."
-	@rm tex/*
+	@-rm ../latex/skeleton/*.tplx

IPython/nbconvert/templates/skeleton/README.md

@@ -1,6 +1,8 b''
 ## Template skeleton
 
-This contain skeleton template that you probably don't want
-to inherit directly.
+This directory contains the template skeleton files.
 
-do not modify the content of the 'tex' folder which is generated by running 'make' in this folder.
+Do not modify the contents of the `../latex/skeleton` folder.  Instead, 
+if you need to, make modifications to the files in this folder and then run
+`make` to generate the corresponding latex skeleton files in the 
+`../latex/skeleton` folder.
  No newline at end of file

IPython/nbconvert/templates/skeleton/display_priority.tpl

@@ -29,10 +29,13 b''
             {%- block data_text -%}
             {%- endblock -%}
         {%- endif -%}
-
         {%- if type in ['latex']%}
             {%- block data_latex -%}
             {%- endblock -%}
         {%- endif -%}
+        {%- if type in ['javascript']%}
+            {%- block data_javascript -%}
+            {%- endblock -%}
+        {%- endif -%}
     {%- endfor -%}
 {%- endblock data_priority -%}

IPython/nbconvert/templates/skeleton/null.tpl

@@ -1,10 +1,10 b''
 {#
 
-DO NOT USE THIS AS A BASE WORK,
+DO NOT USE THIS AS A BASE,
 IF YOU ARE COPY AND PASTING THIS FILE
-YOU ARE PROBABLY DOING THINGS WRONG.
+YOU ARE PROBABLY DOING THINGS INCORRECTLY.
 
-Null template, Does nothing except defining a basic structure
+Null template, does nothing except defining a basic structure
 To layout the different blocks of a notebook.
 
 Subtemplates can override blocks to define their custom representation.

IPython/nbconvert/templates/slides_reveal.tpl

@@ -135,7 +135,7 b" transition: Reveal.getQueryHash().transition || 'linear', // default/cube/page/c"
 dependencies: [
 { src: "{{resources.reveal.url_prefix}}/lib/js/classList.js", condition: function() { return !document.body.classList; } },
 { src: "{{resources.reveal.url_prefix}}/plugin/highlight/highlight.js", async: true, callback: function() { hljs.initHighlightingOnLoad(); } },
-{ src: "{{resources.reveal.notes_prefix}}/plugin/notes/notes.js", async: true, condition: function() { return !!document.body.classList; } }
+{ src: "{{resources.reveal.url_prefix}}/plugin/notes/notes.js", async: true, condition: function() { return !!document.body.classList; } }
 // { src: 'http://s7.addthis.com/js/300/addthis_widget.js', async: true},
 ]
 });

IPython/nbconvert/tests/base.py

@@ -16,12 +16,12 b' Contains base test class for nbconvert'
 import os
 import glob
 import shutil
+import unittest
 
 import IPython
 from IPython.utils.tempdir import TemporaryWorkingDirectory
 from IPython.utils.process import get_output_error_code
 from IPython.testing.tools import get_ipython_cmd
-from IPython.testing.ipunittest import ParametricTestCase
 
 # a trailing space allows for simpler concatenation with the other arguments
 ipy_cmd = get_ipython_cmd(as_string=True) + " "
@@ -31,7 +31,7 b' ipy_cmd = get_ipython_cmd(as_string=True) + " "'
 #-----------------------------------------------------------------------------
 
 
-class TestsBase(ParametricTestCase):
+class TestsBase(unittest.TestCase):
     """Base tests class.  Contains useful fuzzy comparison and nbconvert
     functions."""
 
@@ -86,7 +86,7 b' class TestsBase(ParametricTestCase):'
 
         For example:
            Replace "ii" with "i" in the string "Hiiii" yields "Hii"
-           Another replacement yields "Hi" (the desired output)
+           Another replacement cds "Hi" (the desired output)
 
         Parameters:
         -----------

IPython/nbconvert/tests/files/notebook2.ipynb

@@ -124,6 +124,14 b''
      "prompt_number": 10
     },
     {
+     "cell_type": "heading",
+     "level": 2,
+     "metadata": {},
+     "source": [
+      "Here is a very long heading that pandoc will wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap"
+     ]
+    },
+    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [

IPython/nbconvert/tests/test_nbconvertapp.py

@@ -1,12 +1,10 b''
-"""
-Contains tests for the nbconvertapp
-"""
+"""Test NbConvertApp"""
+
 #-----------------------------------------------------------------------------
-#Copyright (c) 2013, the IPython Development Team.
-#
-#Distributed under the terms of the Modified BSD License.
+#  Copyright (C) 2013 The IPython Development Team
 #
-#The full license is in the file COPYING.txt, distributed with this software.
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
@@ -15,12 +13,14 b' Contains tests for the nbconvertapp'
 
 import os
 import glob
+import sys
 
 from .base import TestsBase
 
+import IPython.testing.tools as tt
 from IPython.testing import decorators as dec
 
-    
+
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
@@ -35,13 +35,14 b' class TestNbConvertApp(TestsBase):'
 
 
     def test_notebook_help(self):
-        """
-        Will help show if no notebooks are specified?
-        """
+        """Will help show if no notebooks are specified?"""
         with self.create_temp_cwd():
             out, err = self.call('nbconvert --log-level 0', ignore_return_code=True)
-            assert "see '--help-all'" in out
-
+            self.assertIn("see '--help-all'", out)
+    
+    def test_help_output(self):
+        """ipython nbconvert --help-all works"""
+        tt.help_all_output_test('nbconvert')
 
     def test_glob(self):
         """
@@ -102,6 +103,30 b' class TestNbConvertApp(TestsBase):'
             assert os.path.isfile('notebook1.tex')
             assert os.path.isfile('notebook1.pdf')
 
+    @dec.onlyif_cmds_exist('pandoc')
+    def test_spurious_cr(self):
+        """Check for extra CR characters"""
+        with self.create_temp_cwd(['notebook2.ipynb']):
+            self.call('nbconvert --log-level 0 --to latex notebook2')
+            assert os.path.isfile('notebook2.tex')
+            with open('notebook2.tex') as f:
+                tex = f.read()
+            self.call('nbconvert --log-level 0 --to html notebook2')
+            assert os.path.isfile('notebook2.html')
+            with open('notebook2.html') as f:
+                html = f.read()
+        self.assertEqual(tex.count('\r'), tex.count('\r\n'))
+        self.assertEqual(html.count('\r'), html.count('\r\n'))
+
+    @dec.onlyif_cmds_exist('pandoc')
+    def test_png_base64_html_ok(self):
+        """Is embedded png data well formed in HTML?"""
+        with self.create_temp_cwd(['notebook2.ipynb']):
+            self.call('nbconvert --log-level 0 --to HTML '
+                      'notebook2.ipynb --template full')
+            assert os.path.isfile('notebook2.html')
+            with open('notebook2.html') as f:
+                assert "data:image/png;base64,b'" not in f.read()
 
     @dec.onlyif_cmds_exist('pandoc')
     def test_template(self):

IPython/nbconvert/utils/pandoc.py

@@ -15,6 +15,7 b' from __future__ import print_function'
 
 # Stdlib imports
 import subprocess
+from io import TextIOWrapper, BytesIO
 
 # IPython imports
 from IPython.utils.py3compat import cast_bytes
@@ -64,6 +65,6 b" def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'):"
             "http://johnmacfarlane.net/pandoc/installing.html"
         )
     out, _ = p.communicate(cast_bytes(source, encoding))
-    out = out.decode(encoding, 'replace')
-    return out[:-1]
+    out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
+    return out.rstrip('\n')
 

IPython/nbformat/v3/nbbase.py

@@ -25,6 +25,7 b' import pprint'
 import uuid
 
 from IPython.utils.ipstruct import Struct
+from IPython.utils.py3compat import cast_unicode
 
 #-----------------------------------------------------------------------------
 # Code
@@ -67,21 +68,21 b' def new_output(output_type=None, output_text=None, output_png=None,'
 
     if output_type != 'pyerr':
         if output_text is not None:
-            output.text = unicode(output_text)
+            output.text = cast_unicode(output_text)
         if output_png is not None:
-            output.png = bytes(output_png)
+            output.png = cast_unicode(output_png)
         if output_jpeg is not None:
-            output.jpeg = bytes(output_jpeg)
+            output.jpeg = cast_unicode(output_jpeg)
         if output_html is not None:
-            output.html = unicode(output_html)
+            output.html = cast_unicode(output_html)
         if output_svg is not None:
-            output.svg = unicode(output_svg)
+            output.svg = cast_unicode(output_svg)
         if output_latex is not None:
-            output.latex = unicode(output_latex)
+            output.latex = cast_unicode(output_latex)
         if output_json is not None:
-            output.json = unicode(output_json)
+            output.json = cast_unicode(output_json)
         if output_javascript is not None:
-            output.javascript = unicode(output_javascript)
+            output.javascript = cast_unicode(output_javascript)
 
     if output_type == u'pyout':
         if prompt_number is not None:
@@ -89,14 +90,14 b' def new_output(output_type=None, output_text=None, output_png=None,'
 
     if output_type == u'pyerr':
         if ename is not None:
-            output.ename = unicode(ename)
+            output.ename = cast_unicode(ename)
         if evalue is not None:
-            output.evalue = unicode(evalue)
+            output.evalue = cast_unicode(evalue)
         if traceback is not None:
-            output.traceback = [unicode(frame) for frame in list(traceback)]
+            output.traceback = [cast_unicode(frame) for frame in list(traceback)]
 
     if output_type == u'stream':
-        output.stream = 'stdout' if stream is None else unicode(stream)
+        output.stream = 'stdout' if stream is None else cast_unicode(stream)
     
     return output
 
@@ -107,9 +108,9 b' def new_code_cell(input=None, prompt_number=None, outputs=None,'
     cell = NotebookNode()
     cell.cell_type = u'code'
     if language is not None:
-        cell.language = unicode(language)
+        cell.language = cast_unicode(language)
     if input is not None:
-        cell.input = unicode(input)
+        cell.input = cast_unicode(input)
     if prompt_number is not None:
         cell.prompt_number = int(prompt_number)
     if outputs is None:
@@ -130,9 +131,9 b' def new_text_cell(cell_type, source=None, rendered=None, metadata=None):'
     if cell_type == 'plaintext':
         cell_type = 'raw'
     if source is not None:
-        cell.source = unicode(source)
+        cell.source = cast_unicode(source)
     if rendered is not None:
-        cell.rendered = unicode(rendered)
+        cell.rendered = cast_unicode(rendered)
     cell.metadata = NotebookNode(metadata or {})
     cell.cell_type = cell_type
     return cell
@@ -143,9 +144,9 b' def new_heading_cell(source=None, rendered=None, level=1, metadata=None):'
     cell = NotebookNode()
     cell.cell_type = u'heading'
     if source is not None:
-        cell.source = unicode(source)
+        cell.source = cast_unicode(source)
     if rendered is not None:
-        cell.rendered = unicode(rendered)
+        cell.rendered = cast_unicode(rendered)
     cell.level = int(level)
     cell.metadata = NotebookNode(metadata or {})
     return cell
@@ -155,7 +156,7 b' def new_worksheet(name=None, cells=None, metadata=None):'
     """Create a worksheet by name with with a list of cells."""
     ws = NotebookNode()
     if name is not None:
-        ws.name = unicode(name)
+        ws.name = cast_unicode(name)
     if cells is None:
         ws.cells = []
     else:
@@ -178,7 +179,7 b' def new_notebook(name=None, metadata=None, worksheets=None):'
     else:
         nb.metadata = NotebookNode(metadata)
     if name is not None:
-        nb.metadata.name = unicode(name)
+        nb.metadata.name = cast_unicode(name)
     return nb
 
 
@@ -187,29 +188,29 b' def new_metadata(name=None, authors=None, license=None, created=None,'
     """Create a new metadata node."""
     metadata = NotebookNode()
     if name is not None:
-        metadata.name = unicode(name)
+        metadata.name = cast_unicode(name)
     if authors is not None:
         metadata.authors = list(authors)
     if created is not None:
-        metadata.created = unicode(created)
+        metadata.created = cast_unicode(created)
     if modified is not None:
-        metadata.modified = unicode(modified)
+        metadata.modified = cast_unicode(modified)
     if license is not None:
-        metadata.license = unicode(license)
+        metadata.license = cast_unicode(license)
     if gistid is not None:
-        metadata.gistid = unicode(gistid)
+        metadata.gistid = cast_unicode(gistid)
     return metadata
 
 def new_author(name=None, email=None, affiliation=None, url=None):
     """Create a new author."""
     author = NotebookNode()
     if name is not None:
-        author.name = unicode(name)
+        author.name = cast_unicode(name)
     if email is not None:
-        author.email = unicode(email)
+        author.email = cast_unicode(email)
     if affiliation is not None:
-        author.affiliation = unicode(affiliation)
+        author.affiliation = cast_unicode(affiliation)
     if url is not None:
-        author.url = unicode(url)
+        author.url = cast_unicode(url)
     return author
 

IPython/nbformat/v3/nbjson.py

@@ -46,7 +46,7 b' class JSONReader(NotebookReader):'
         return nb
 
     def to_notebook(self, d, **kwargs):
-        return restore_bytes(rejoin_lines(from_dict(d)))
+        return rejoin_lines(from_dict(d))
 
 
 class JSONWriter(NotebookWriter):

IPython/nbformat/v3/nbpy.py

@@ -190,7 +190,7 b' class PyWriter(NotebookWriter):'
                         lines.extend([u'# ' + line for line in input.splitlines()])
                         lines.append(u'')
         lines.append('')
-        return unicode('\n'.join(lines))
+        return u'\n'.join(lines)
 
 
 _reader = PyReader()

IPython/nbformat/v3/rwbase.py

@@ -32,6 +32,8 b' def restore_bytes(nb):'
     
     Base64 encoding is handled elsewhere.  Bytes objects in the notebook are
     always b64-encoded. We DO NOT encode/decode around file formats.
+    
+    Note: this is never used
     """
     for ws in nb.worksheets:
         for cell in ws.cells:

IPython/nbformat/v3/tests/nbexamples.py

@@ -9,9 +9,9 b' from ..nbbase import ('
     new_metadata, new_author, new_heading_cell, nbformat, nbformat_minor
 )
 
-# some random base64-encoded *bytes*
-png = encodestring(os.urandom(5))
-jpeg = encodestring(os.urandom(6))
+# some random base64-encoded *text*
+png = encodestring(os.urandom(5)).decode('ascii')
+jpeg = encodestring(os.urandom(6)).decode('ascii')
 
 ws = new_worksheet(name='worksheet1')
 

IPython/nbformat/v3/tests/test_json.py

@@ -1,4 +1,5 b''
 import pprint
+from base64 import decodestring
 from unittest import TestCase
 
 from ..nbjson import reads, writes
@@ -29,5 +30,42 b' class TestJSON(formattest.NBFormatTest, TestCase):'
         s = writes(nb0, split_lines=True)
         self.assertEqual(nbjson.reads(s),nb0)
 
+    def test_read_png(self):
+        """PNG output data is b64 unicode"""
+        s = writes(nb0)
+        nb1 = nbjson.reads(s)
+        found_png = False
+        for cell in nb1.worksheets[0].cells:
+            if not 'outputs' in cell:
+                continue
+            for output in cell.outputs:
+                if 'png' in output:
+                    found_png = True
+                    pngdata = output['png']
+                    self.assertEqual(type(pngdata), unicode)
+                    # test that it is valid b64 data
+                    b64bytes = pngdata.encode('ascii')
+                    raw_bytes = decodestring(b64bytes)
+        assert found_png, "never found png output"
+
+    def test_read_jpeg(self):
+        """JPEG output data is b64 unicode"""
+        s = writes(nb0)
+        nb1 = nbjson.reads(s)
+        found_jpeg = False
+        for cell in nb1.worksheets[0].cells:
+            if not 'outputs' in cell:
+                continue
+            for output in cell.outputs:
+                if 'jpeg' in output:
+                    found_jpeg = True
+                    jpegdata = output['jpeg']
+                    self.assertEqual(type(jpegdata), unicode)
+                    # test that it is valid b64 data
+                    b64bytes = jpegdata.encode('ascii')
+                    raw_bytes = decodestring(b64bytes)
+        assert found_jpeg, "never found jpeg output"
+
+
 
 

IPython/nbformat/v3/tests/test_nbbase.py

@@ -141,3 +141,17 b' class TestMetadata(TestCase):'
         self.assertEqual(md.gistid, u'21341231')
         self.assertEqual(md.authors, authors)
 
+class TestOutputs(TestCase):
+    def test_binary_png(self):
+        out = new_output(output_png=b'\x89PNG\r\n\x1a\n')
+
+    def test_b64b6tes_png(self):
+        out = new_output(output_png=b'iVBORw0KG')
+    
+    def test_binary_jpeg(self):
+        out = new_output(output_jpeg=b'\xff\xd8')
+
+    def test_b64b6tes_jpeg(self):
+        out = new_output(output_jpeg=b'/9')
+        
+

IPython/nbformat/v3/validator.py

@@ -1,11 +1,11 b''
 #!/usr/bin/env python
 # -*- coding: utf8 -*-
+import argparse
+import traceback
+import json
 
 from IPython.external.jsonschema import  Draft3Validator, validate, ValidationError
 import IPython.external.jsonpointer as jsonpointer
-from IPython.external import argparse
-import traceback
-import json
 
 def nbvalidate(nbjson, schema='v3.withref.json', key=None,verbose=True):
     v3schema = resolve_ref(json.load(open(schema,'r')))

IPython/parallel/apps/baseapp.py

@@ -71,7 +71,6 b' class ParallelCrashHandler(CrashHandler):'
 base_aliases = {}
 base_aliases.update(base_ip_aliases)
 base_aliases.update({
-    'profile-dir' : 'ProfileDir.location',
     'work-dir' : 'BaseParallelApplication.work_dir',
     'log-to-file' : 'BaseParallelApplication.log_to_file',
     'clean-logs' : 'BaseParallelApplication.clean_logs',

IPython/parallel/apps/ipcontrollerapp.py

@@ -11,7 +11,7 b' Authors:'
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2008  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
@@ -44,7 +44,7 b' from IPython.parallel.apps.baseapp import ('
     catch_config_error,
 )
 from IPython.utils.importstring import import_item
-from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS
+from IPython.utils.localinterfaces import localhost, public_ips
 from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError
 
 from IPython.kernel.zmq.session import (
@@ -224,13 +224,13 b' class IPControllerApp(BaseParallelApplication):'
         location = cdict['location']
         
         if not location:
-            if PUBLIC_IPS:
-                location = PUBLIC_IPS[-1]
+            if public_ips():
+                location = public_ips()[-1]
             else:
                 self.log.warn("Could not identify this machine's IP, assuming %s."
                 " You may need to specify '--location=<external_ip_address>' to help"
-                " IPython decide when to connect via loopback." % LOCALHOST)
-                location = LOCALHOST
+                " IPython decide when to connect via loopback." % localhost() )
+                location = localhost()
             cdict['location'] = location
         fname = os.path.join(self.profile_dir.security_dir, fname)
         self.log.info("writing connection info to %s", fname)

IPython/parallel/apps/logwatcher.py

@@ -26,7 +26,7 b' import zmq'
 from zmq.eventloop import ioloop, zmqstream
 
 from IPython.config.configurable import LoggingConfigurable
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils.traitlets import Int, Unicode, Instance, List
 
 #-----------------------------------------------------------------------------
@@ -44,8 +44,10 b' class LogWatcher(LoggingConfigurable):'
     # configurables
     topics = List([''], config=True,
         help="The ZMQ topics to subscribe to. Default is to subscribe to all messages")
-    url = Unicode('tcp://%s:20202' % LOCALHOST, config=True,
+    url = Unicode(config=True,
         help="ZMQ url on which to listen for log messages")
+    def _url_default(self):
+        return 'tcp://%s:20202' % localhost()
     
     # internals
     stream = Instance('zmq.eventloop.zmqstream.ZMQStream')

IPython/parallel/client/asyncresult.py

@@ -391,7 +391,7 b' class AsyncResult(object):'
         tic = time.time()
         while not self.ready() and (timeout < 0 or time.time() - tic <= timeout):
             self.wait(interval)
-            clear_output()
+            clear_output(wait=True)
             print("%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed), end="")
             sys.stdout.flush()
         print()

IPython/parallel/client/client.py

@@ -37,7 +37,7 b' from IPython.core.profiledir import ProfileDir, ProfileDirError'
 from IPython.utils.capture import RichOutput
 from IPython.utils.coloransi import TermColors
 from IPython.utils.jsonutil import rekey
-from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS
+from IPython.utils.localinterfaces import localhost, is_local_ip
 from IPython.utils.path import get_ipython_dir
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode,
@@ -433,13 +433,13 b' class Client(HasTraits):'
         
         url = cfg['registration']
         
-        if location is not None and addr == LOCALHOST:
+        if location is not None and addr == localhost():
             # location specified, and connection is expected to be local
-            if location not in LOCAL_IPS and not sshserver:
+            if not is_local_ip(location) and not sshserver:
                 # load ssh from JSON *only* if the controller is not on
                 # this machine
                 sshserver=cfg['ssh']
-            if location not in LOCAL_IPS and not sshserver:
+            if not is_local_ip(location) and not sshserver:
                 # warn if no ssh specified, but SSH is probably needed
                 # This is only a warning, because the most likely cause
                 # is a local Controller on a laptop whose IP is dynamic
@@ -495,7 +495,12 b' class Client(HasTraits):'
                                     }
         self._queue_handlers = {'execute_reply' : self._handle_execute_reply,
                                 'apply_reply' : self._handle_apply_reply}
-        self._connect(sshserver, ssh_kwargs, timeout)
+        
+        try:
+            self._connect(sshserver, ssh_kwargs, timeout)
+        except:
+            self.close(linger=0)
+            raise
         
         # last step: setup magics, if we are in IPython:
         
@@ -599,7 +604,6 b' class Client(HasTraits):'
         self._connected=True
 
         def connect_socket(s, url):
-            # url = util.disambiguate_url(url, self._config['location'])
             if self._ssh:
                 return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs)
             else:
@@ -956,14 +960,23 b' class Client(HasTraits):'
         view.activate(suffix)
         return view
 
-    def close(self):
+    def close(self, linger=None):
+        """Close my zmq Sockets
+        
+        If `linger`, set the zmq LINGER socket option,
+        which allows discarding of messages.
+        """
         if self._closed:
             return
         self.stop_spin_thread()
-        snames = filter(lambda n: n.endswith('socket'), dir(self))
-        for socket in map(lambda name: getattr(self, name), snames):
-            if isinstance(socket, zmq.Socket) and not socket.closed:
-                socket.close()
+        snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ]
+        for name in snames:
+            socket = getattr(self, name)
+            if socket is not None and not socket.closed:
+                if linger is not None:
+                    socket.close(linger=linger)
+                else:
+                    socket.close()
         self._closed = True
 
     def _spin_every(self, interval=1):

IPython/parallel/client/remotefunction.py

@@ -89,7 +89,6 b' def sync_view_results(f, self, *args, **kwargs):'
     view = self.view
     if view._in_sync_results:
         return f(self, *args, **kwargs)
-    print 'in sync results', f
     view._in_sync_results = True
     try:
         ret = f(self, *args, **kwargs)

IPython/parallel/controller/hub.py

@@ -30,7 +30,7 b' from zmq.eventloop.zmqstream import ZMQStream'
 
 # internal:
 from IPython.utils.importstring import import_item
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.traitlets import (
         HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName
@@ -177,20 +177,25 b' class HubFactory(RegistrationFactory):'
     def _notifier_port_default(self):
         return util.select_random_ports(1)[0]
 
-    engine_ip = Unicode(LOCALHOST, config=True,
+    engine_ip = Unicode(config=True,
         help="IP on which to listen for engine connections. [default: loopback]")
+    def _engine_ip_default(self):
+        return localhost()
     engine_transport = Unicode('tcp', config=True,
         help="0MQ transport for engine connections. [default: tcp]")
 
-    client_ip = Unicode(LOCALHOST, config=True,
+    client_ip = Unicode(config=True,
         help="IP on which to listen for client connections. [default: loopback]")
     client_transport = Unicode('tcp', config=True,
         help="0MQ transport for client connections. [default : tcp]")
 
-    monitor_ip = Unicode(LOCALHOST, config=True,
+    monitor_ip = Unicode(config=True,
         help="IP on which to listen for monitor messages. [default: loopback]")
     monitor_transport = Unicode('tcp', config=True,
         help="0MQ transport for monitor messages. [default : tcp]")
+    
+    _client_ip_default = _monitor_ip_default = _engine_ip_default
+    
 
     monitor_url = Unicode('')
 

IPython/parallel/controller/mongodb.py

@@ -39,7 +39,7 b' class MongoDB(BaseDB):'
         necessary if the default mongodb configuration does not point to your
         mongod instance."""
     )
-    database = Unicode(config=True,
+    database = Unicode("ipython-tasks", config=True,
         help="""The MongoDB database name to use for storing tasks for this session. If unspecified,
         a new database will be created with the Hub's IDENT.  Specifying the database will result
         in tasks from previous sessions being available via Clients' db_query and

IPython/parallel/controller/scheduler.py

@@ -358,7 +358,7 b' class TaskScheduler(SessionFactory):'
             # build fake metadata
             md = dict(
                 status=u'error',
-                engine=engine,
+                engine=engine.decode('ascii'),
                 date=datetime.now(),
             )
             msg = self.session.msg('apply_reply', content, parent=parent, metadata=md)

IPython/parallel/controller/sqlitedb.py

@@ -96,7 +96,7 b' class SQLiteDB(BaseDB):'
     location = Unicode('', config=True,
         help="""The directory containing the sqlite task database.  The default
         is to use the cluster_dir location.""")
-    table = Unicode("", config=True,
+    table = Unicode("ipython-tasks", config=True,
         help="""The SQLite Table to use for storing tasks for this session. If unspecified,
         a new table will be created with the Hub's IDENT.  Specifying the table will result
         in tasks from previous sessions being available via Clients' db_query and
@@ -195,7 +195,7 b' class SQLiteDB(BaseDB):'
 
         If a bad (old) table does exist, return False
         """
-        cursor = self._db.execute("PRAGMA table_info(%s)"%self.table)
+        cursor = self._db.execute("PRAGMA table_info('%s')"%self.table)
         lines = cursor.fetchall()
         if not lines:
             # table does not exist
@@ -241,7 +241,7 b' class SQLiteDB(BaseDB):'
             )
             previous_table = self.table
 
-        self._db.execute("""CREATE TABLE IF NOT EXISTS %s
+        self._db.execute("""CREATE TABLE IF NOT EXISTS '%s'
                 (msg_id text PRIMARY KEY,
                 header dict text,
                 metadata dict text,
@@ -333,12 +333,12 b' class SQLiteDB(BaseDB):'
         d['msg_id'] = msg_id
         line = self._dict_to_list(d)
         tups = '(%s)'%(','.join(['?']*len(line)))
-        self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line)
+        self._db.execute("INSERT INTO '%s' VALUES %s"%(self.table, tups), line)
         # self._db.commit()
 
     def get_record(self, msg_id):
         """Get a specific Task Record, by msg_id."""
-        cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
+        cursor = self._db.execute("""SELECT * FROM '%s' WHERE msg_id==?"""%self.table, (msg_id,))
         line = cursor.fetchone()
         if line is None:
             raise KeyError("No such msg: %r"%msg_id)
@@ -346,7 +346,7 b' class SQLiteDB(BaseDB):'
 
     def update_record(self, msg_id, rec):
         """Update the data in an existing record."""
-        query = "UPDATE %s SET "%self.table
+        query = "UPDATE '%s' SET "%self.table
         sets = []
         keys = sorted(rec.keys())
         values = []
@@ -361,13 +361,13 b' class SQLiteDB(BaseDB):'
 
     def drop_record(self, msg_id):
         """Remove a record from the DB."""
-        self._db.execute("""DELETE FROM %s WHERE msg_id==?"""%self.table, (msg_id,))
+        self._db.execute("""DELETE FROM '%s' WHERE msg_id==?"""%self.table, (msg_id,))
         # self._db.commit()
 
     def drop_matching_records(self, check):
         """Remove a record from the DB."""
         expr,args = self._render_expression(check)
-        query = "DELETE FROM %s WHERE %s"%(self.table, expr)
+        query = "DELETE FROM '%s' WHERE %s"%(self.table, expr)
         self._db.execute(query,args)
         # self._db.commit()
 
@@ -399,7 +399,7 b' class SQLiteDB(BaseDB):'
         else:
             req = '*'
         expr,args = self._render_expression(check)
-        query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr)
+        query = """SELECT %s FROM '%s' WHERE %s"""%(req, self.table, expr)
         cursor = self._db.execute(query, args)
         matches = cursor.fetchall()
         records = []
@@ -410,7 +410,7 b' class SQLiteDB(BaseDB):'
 
     def get_history(self):
         """get all msg_ids, ordered by time submitted."""
-        query = """SELECT msg_id FROM %s ORDER by submitted ASC"""%self.table
+        query = """SELECT msg_id FROM '%s' ORDER by submitted ASC"""%self.table
         cursor = self._db.execute(query)
         # will be a list of length 1 tuples
         return [ tup[0] for tup in cursor.fetchall()]

IPython/parallel/engine/engine.py

@@ -24,7 +24,7 b' from zmq.eventloop import ioloop, zmqstream'
 
 from IPython.external.ssh import tunnel
 # internal
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils.traitlets import (
     Instance, Dict, Integer, Type, Float, Integer, Unicode, CBytes, Bool
 )
@@ -184,13 +184,13 b' class EngineFactory(RegistrationFactory):'
             if self.max_heartbeat_misses > 0:
                 # Add a monitor socket which will record the last time a ping was seen
                 mon = self.context.socket(zmq.SUB)
-                mport = mon.bind_to_random_port('tcp://%s' % LOCALHOST)
+                mport = mon.bind_to_random_port('tcp://%s' % localhost())
                 mon.setsockopt(zmq.SUBSCRIBE, b"")
                 self._hb_listener = zmqstream.ZMQStream(mon, self.loop)
                 self._hb_listener.on_recv(self._report_ping)
             
             
-                hb_monitor = "tcp://%s:%i" % (LOCALHOST, mport)
+                hb_monitor = "tcp://%s:%i" % (localhost(), mport)
 
             heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity)
             heart.start()

IPython/parallel/factory.py

@@ -24,7 +24,7 b' import zmq'
 from zmq.eventloop.ioloop import IOLoop
 
 from IPython.config.configurable import Configurable
-from IPython.utils.localinterfaces import LOCALHOST
+from IPython.utils.localinterfaces import localhost
 from IPython.utils.traitlets import Integer, Instance, Unicode
 
 from IPython.parallel.util import select_random_ports
@@ -40,16 +40,18 b' class RegistrationFactory(SessionFactory):'
     
     url = Unicode('', config=True,
         help="""The 0MQ url used for registration. This sets transport, ip, and port
-        in one variable. For example: url='tcp://%s:12345' or
+        in one variable. For example: url='tcp://127.0.0.1:12345' or
         url='epgm://*:90210'"""
-                  % LOCALHOST) # url takes precedence over ip,regport,transport
+        ) # url takes precedence over ip,regport,transport
     transport = Unicode('tcp', config=True,
         help="""The 0MQ transport for communications.  This will likely be
         the default of 'tcp', but other values include 'ipc', 'epgm', 'inproc'.""")
-    ip = Unicode(LOCALHOST, config=True,
+    ip = Unicode(config=True,
         help="""The IP address for registration.  This is generally either
         '127.0.0.1' for loopback only or '*' for all interfaces.
-        [default: '%s']""" % LOCALHOST)
+        """)
+    def _ip_default(self):
+        return localhost()
     regport = Integer(config=True,
         help="""The port on which the Hub listens for registration.""")
     def _regport_default(self):

IPython/parallel/tests/__init__.py

@@ -63,8 +63,7 b' def setup():'
     tic = time.time()
     while not os.path.exists(engine_json) or not os.path.exists(client_json):
         if cp.poll() is not None:
-            print cp.poll()
-            raise RuntimeError("The test controller failed to start.")
+            raise RuntimeError("The test controller exited with status %s" % cp.poll())
         elif time.time()-tic > 15:
             raise RuntimeError("Timeout waiting for the test controller to start.")
         time.sleep(0.1)

IPython/parallel/tests/test_client.py

@@ -172,7 +172,6 b' class TestClient(ClusterTestCase):'
         # give the monitor time to notice the message
         time.sleep(.25)
         ahr = self.client.get_result(ar.msg_ids[0])
-        print ar.get(), ahr.get(), ar._single_result, ahr._single_result
         self.assertTrue(isinstance(ahr, AsyncHubResult))
         self.assertEqual(ahr.get().pyout, ar.get().pyout)
         ar2 = self.client.get_result(ar.msg_ids[0])