Merge pull request #4778 from minrk/install-nbextensions

add APIs for installing notebook extensions

IPython/html/nbextensions.py

@@ -0,0 +1,268 b''
+# coding: utf-8
+"""Utilities for installing Javascript extensions for the notebook"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2014 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function
+
+import os
+import shutil
+import tarfile
+import zipfile
+from os.path import basename, join as pjoin
+
+# Deferred imports
+try:
+    from urllib.parse import urlparse  # Py3
+    from urllib.request import urlretrieve
+except ImportError:
+    from urlparse import urlparse
+    from urllib import urlretrieve
+
+from IPython.utils.path import get_ipython_dir
+from IPython.utils.py3compat import string_types, cast_unicode_py2
+from IPython.utils.tempdir import TemporaryDirectory
+
+
+def _should_copy(src, dest, verbose=1):
+    """should a file be copied?"""
+    if not os.path.exists(dest):
+        return True
+    if os.stat(dest).st_mtime < os.stat(src).st_mtime:
+        if verbose >= 2:
+            print("%s is out of date" % dest)
+        return True
+    if verbose >= 2:
+        print("%s is up to date" % dest)
+    return False
+
+
+def _maybe_copy(src, dest, verbose=1):
+    """copy a file if it needs updating"""
+    if _should_copy(src, dest, verbose):
+        if verbose >= 1:
+            print("copying %s -> %s" % (src, dest))
+        shutil.copy2(src, dest)
+
+
+def _safe_is_tarfile(path):
+    """safe version of is_tarfile, return False on IOError"""
+    try:
+        return tarfile.is_tarfile(path)
+    except IOError:
+        return False
+
+
+def check_nbextension(files, ipython_dir=None):
+    """Check whether nbextension files have been installed
+    
+    files should be a list of relative paths within nbextensions.
+    
+    Returns True if all files are found, False if any are missing.
+    """
+    ipython_dir = ipython_dir or get_ipython_dir()
+    nbext = pjoin(ipython_dir, u'nbextensions')
+    # make sure nbextensions dir exists
+    if not os.path.exists(nbext):
+        return False
+    
+    if isinstance(files, string_types):
+        # one file given, turn it into a list
+        files = [files]
+    
+    return all(os.path.exists(pjoin(nbext, f)) for f in files)
+
+
+def install_nbextension(files, overwrite=False, symlink=False, ipython_dir=None, verbose=1):
+    """Install a Javascript extension for the notebook
+    
+    Stages files and/or directories into IPYTHONDIR/nbextensions.
+    By default, this compares modification time, and only stages files that need updating.
+    If `overwrite` is specified, matching files are purged before proceeding.
+    
+    Parameters
+    ----------
+    
+    files : list(paths or URLs)
+        One or more paths or URLs to existing files directories to install.
+        These will be installed with their base name, so '/path/to/foo'
+        will install to 'nbextensions/foo'.
+        Archives (zip or tarballs) will be extracted into the nbextensions directory.
+    overwrite : bool [default: False]
+        If True, always install the files, regardless of what may already be installed.
+    symlink : bool [default: False]
+        If True, create a symlink in nbextensions, rather than copying files.
+        Not allowed with URLs or archives.
+    ipython_dir : str [optional]
+        The path to an IPython directory, if the default value is not desired.
+        get_ipython_dir() is used by default.
+    verbose : int [default: 1]
+        Set verbosity level. The default is 1, where file actions are printed.
+        set verbose=2 for more output, or verbose=0 for silence.
+    """
+    
+    ipython_dir = ipython_dir or get_ipython_dir()
+    nbext = pjoin(ipython_dir, u'nbextensions')
+    # make sure nbextensions dir exists
+    if not os.path.exists(nbext):
+        os.makedirs(nbext)
+    
+    if isinstance(files, string_types):
+        # one file given, turn it into a list
+        files = [files]
+    
+    for path in map(cast_unicode_py2, files):
+        
+        if path.startswith(('https://', 'http://')):
+            if symlink:
+                raise ValueError("Cannot symlink from URLs")
+            # Given a URL, download it
+            with TemporaryDirectory() as td:
+                filename = urlparse(path).path.split('/')[-1]
+                local_path = os.path.join(td, filename)
+                if verbose >= 1:
+                    print("downloading %s to %s" % (path, local_path))
+                urlretrieve(path, local_path)
+                # now install from the local copy
+                install_nbextension(local_path, overwrite, symlink, ipython_dir, verbose)
+            continue
+        
+        # handle archives
+        archive = None
+        if path.endswith('.zip'):
+            archive = zipfile.ZipFile(path)
+        elif _safe_is_tarfile(path):
+            archive = tarfile.open(path)
+        
+        if archive:
+            if symlink:
+                raise ValueError("Cannot symlink from archives")
+            if verbose >= 1:
+                print("extracting %s to %s" % (path, nbext))
+            archive.extractall(nbext)
+            archive.close()
+            continue
+        
+        dest = pjoin(nbext, basename(path))
+        if overwrite and os.path.exists(dest):
+            if verbose >= 1:
+                print("removing %s" % dest)
+            if os.path.isdir(dest):
+                shutil.rmtree(dest)
+            else:
+                os.remove(dest)
+        
+        if symlink:
+            path = os.path.abspath(path)
+            if not os.path.exists(dest):
+                if verbose >= 1:
+                    print("symlink %s -> %s" % (dest, path))
+                os.symlink(path, dest)
+            continue
+
+        if os.path.isdir(path):
+            strip_prefix_len = len(path) - len(basename(path))
+            for parent, dirs, files in os.walk(path):
+                dest_dir = pjoin(nbext, parent[strip_prefix_len:])
+                if not os.path.exists(dest_dir):
+                    if verbose >= 2:
+                        print("making directory %s" % dest_dir)
+                    os.makedirs(dest_dir)
+                for file in files:
+                    src = pjoin(parent, file)
+                    # print("%r, %r" % (dest_dir, file))
+                    dest = pjoin(dest_dir, file)
+                    _maybe_copy(src, dest, verbose)
+        else:
+            src = path
+            _maybe_copy(src, dest, verbose)
+
+#----------------------------------------------------------------------
+# install nbextension app
+#----------------------------------------------------------------------
+
+from IPython.utils.traitlets import Bool, Enum
+from IPython.core.application import BaseIPythonApplication
+
+flags = {
+    "overwrite" : ({
+        "NBExtensionApp" : {
+            "overwrite" : True,
+        }}, "Force overwrite of existing files"
+    ),
+    "debug" : ({
+        "NBExtensionApp" : {
+            "verbose" : 2,
+        }}, "Extra output"
+    ),
+    "quiet" : ({
+        "NBExtensionApp" : {
+            "verbose" : 0,
+        }}, "Minimal output"
+    ),
+    "symlink" : ({
+        "NBExtensionApp" : {
+            "symlink" : True,
+        }}, "Create symlinks instead of copying files"
+    ),
+}
+flags['s'] = flags['symlink']
+
+aliases = {
+    "ipython-dir" : "NBExtensionApp.ipython_dir"
+}
+
+class NBExtensionApp(BaseIPythonApplication):
+    """Entry point for installing notebook extensions"""
+    
+    description = """Install IPython notebook extensions
+    
+    Usage
+    
+        ipython install-nbextension file [more files, folders, archives or urls]
+    
+    This copies files and/or folders into the IPython nbextensions directory.
+    If a URL is given, it will be downloaded.
+    If an archive is given, it will be extracted into nbextensions.
+    If the requested files are already up to date, no action is taken
+    unless --overwrite is specified.
+    """
+    
+    examples = """
+    ipython install-nbextension /path/to/d3.js /path/to/myextension
+    """
+    aliases = aliases
+    flags = flags
+    
+    overwrite = Bool(False, config=True, help="Force overwrite of existing files")
+    symlink = Bool(False, config=True, help="Create symlinks instead of copying files")
+    verbose = Enum((0,1,2), default_value=1, config=True,
+        help="Verbosity level"
+    )
+    
+    def install_extensions(self):
+        install_nbextension(self.extra_args,
+            overwrite=self.overwrite,
+            symlink=self.symlink,
+            verbose=self.verbose,
+            ipython_dir=self.ipython_dir,
+        )
+    
+    def start(self):
+        if not self.extra_args:
+            nbext = pjoin(self.ipython_dir, u'nbextensions')
+            print("Notebook extensions in %s:" % nbext)
+            for ext in os.listdir(nbext):
+                print(u"    %s" % ext)
+        else:
+            self.install_extensions()
+
+
+if __name__ == '__main__':
+    NBExtensionApp.launch_instance()
+    
  No newline at end of file

IPython/html/tests/test_nbextensions.py

@@ -0,0 +1,272 b''
+# coding: utf-8
+"""Test installation of notebook extensions"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2014 The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import glob
+import os
+import re
+import tarfile
+import zipfile
+from io import BytesIO
+from os.path import basename, join as pjoin
+from unittest import TestCase
+
+import IPython.testing.tools as tt
+import IPython.testing.decorators as dec
+from IPython.utils import py3compat
+from IPython.utils.tempdir import TemporaryDirectory
+from IPython.html import nbextensions
+from IPython.html.nbextensions import install_nbextension, check_nbextension
+
+#-----------------------------------------------------------------------------
+# Test functions
+#-----------------------------------------------------------------------------
+
+def touch(file, mtime=None):
+    """ensure a file exists, and set its modification time
+    
+    returns the modification time of the file
+    """
+    open(file, 'a').close()
+    # set explicit mtime
+    if mtime:
+        atime = os.stat(file).st_atime
+        os.utime(file, (atime, mtime))
+    return os.stat(file).st_mtime
+
+
+class TestInstallNBExtension(TestCase):
+    
+    def tempdir(self):
+        td = TemporaryDirectory()
+        self.tempdirs.append(td)
+        return py3compat.cast_unicode(td.name)
+
+    def setUp(self):
+        self.tempdirs = []
+        src = self.src = self.tempdir()
+        self.files = files = [
+            pjoin(u'ƒile'),
+            pjoin(u'∂ir', u'ƒile1'),
+            pjoin(u'∂ir', u'∂ir2', u'ƒile2'),
+        ]
+        for file in files:
+            fullpath = os.path.join(self.src, file)
+            parent = os.path.dirname(fullpath)
+            if not os.path.exists(parent):
+                os.makedirs(parent)
+            touch(fullpath)
+        
+        self.ipdir = self.tempdir()
+        self.save_get_ipython_dir = nbextensions.get_ipython_dir
+        nbextensions.get_ipython_dir = lambda : self.ipdir
+    
+    def tearDown(self):
+        for td in self.tempdirs:
+            td.cleanup()
+        nbextensions.get_ipython_dir = self.save_get_ipython_dir
+    
+    def assert_path_exists(self, path):
+        if not os.path.exists(path):
+            do_exist = os.listdir(os.path.dirname(path))
+            self.fail(u"%s should exist (found %s)" % (path, do_exist))
+    
+    def assert_not_path_exists(self, path):
+        if os.path.exists(path):
+            self.fail(u"%s should not exist" % path)
+    
+    def assert_installed(self, relative_path, ipdir=None):
+        self.assert_path_exists(
+            pjoin(ipdir or self.ipdir, u'nbextensions', relative_path)
+        )
+    
+    def assert_not_installed(self, relative_path, ipdir=None):
+        self.assert_not_path_exists(
+            pjoin(ipdir or self.ipdir, u'nbextensions', relative_path)
+        )
+    
+    def test_create_ipython_dir(self):
+        """install_nbextension when ipython_dir doesn't exist"""
+        with TemporaryDirectory() as td:
+            ipdir = pjoin(td, u'ipython')
+            install_nbextension(self.src, ipython_dir=ipdir)
+            self.assert_path_exists(ipdir)
+            for file in self.files:
+                self.assert_installed(
+                    pjoin(basename(self.src), file),
+                    ipdir
+                )
+    
+    def test_create_nbextensions(self):
+        with TemporaryDirectory() as ipdir:
+            install_nbextension(self.src, ipython_dir=ipdir)
+            self.assert_installed(
+                pjoin(basename(self.src), u'ƒile'),
+                ipdir
+            )
+    
+    def test_single_file(self):
+        file = self.files[0]
+        install_nbextension(pjoin(self.src, file))
+        self.assert_installed(file)
+    
+    def test_single_dir(self):
+        d = u'∂ir'
+        install_nbextension(pjoin(self.src, d))
+        self.assert_installed(self.files[-1])
+    
+    def test_install_nbextension(self):
+        install_nbextension(glob.glob(pjoin(self.src, '*')))
+        for file in self.files:
+            self.assert_installed(file)
+    
+    def test_overwrite_file(self):
+        with TemporaryDirectory() as d:
+            fname = u'ƒ.js'
+            src = pjoin(d, fname)
+            with open(src, 'w') as f:
+                f.write('first')
+            mtime = touch(src)
+            dest = pjoin(self.ipdir, u'nbextensions', fname)
+            install_nbextension(src)
+            with open(src, 'w') as f:
+                f.write('overwrite')
+            mtime = touch(src, mtime - 100)
+            install_nbextension(src, overwrite=True)
+            with open(dest) as f:
+                self.assertEqual(f.read(), 'overwrite')
+    
+    def test_overwrite_dir(self):
+        with TemporaryDirectory() as src:
+            # src = py3compat.cast_unicode_py2(src)
+            base = basename(src)
+            fname = u'ƒ.js'
+            touch(pjoin(src, fname))
+            install_nbextension(src)
+            self.assert_installed(pjoin(base, fname))
+            os.remove(pjoin(src, fname))
+            fname2 = u'∂.js'
+            touch(pjoin(src, fname2))
+            install_nbextension(src, overwrite=True)
+            self.assert_installed(pjoin(base, fname2))
+            self.assert_not_installed(pjoin(base, fname))
+    
+    def test_update_file(self):
+        with TemporaryDirectory() as d:
+            fname = u'ƒ.js'
+            src = pjoin(d, fname)
+            with open(src, 'w') as f:
+                f.write('first')
+            mtime = touch(src)
+            install_nbextension(src)
+            self.assert_installed(fname)
+            dest = pjoin(self.ipdir, u'nbextensions', fname)
+            old_mtime = os.stat(dest).st_mtime
+            with open(src, 'w') as f:
+                f.write('overwrite')
+            touch(src, mtime + 10)
+            install_nbextension(src)
+            with open(dest) as f:
+                self.assertEqual(f.read(), 'overwrite')
+    
+    def test_skip_old_file(self):
+        with TemporaryDirectory() as d:
+            fname = u'ƒ.js'
+            src = pjoin(d, fname)
+            mtime = touch(src)
+            install_nbextension(src)
+            self.assert_installed(fname)
+            dest = pjoin(self.ipdir, u'nbextensions', fname)
+            old_mtime = os.stat(dest).st_mtime
+            
+            mtime = touch(src, mtime - 100)
+            install_nbextension(src)
+            new_mtime = os.stat(dest).st_mtime
+            self.assertEqual(new_mtime, old_mtime)
+
+    def test_quiet(self):
+        with tt.AssertNotPrints(re.compile(r'.+')):
+            install_nbextension(self.src, verbose=0)
+    
+    def test_install_zip(self):
+        path = pjoin(self.src, "myjsext.zip")
+        with zipfile.ZipFile(path, 'w') as f:
+            f.writestr("a.js", b"b();")
+            f.writestr("foo/a.js", b"foo();")
+        install_nbextension(path)
+        self.assert_installed("a.js")
+        self.assert_installed(pjoin("foo", "a.js"))
+    
+    def test_install_tar(self):
+        def _add_file(f, fname, buf):
+            info = tarfile.TarInfo(fname)
+            info.size = len(buf)
+            f.addfile(info, BytesIO(buf))
+        
+        for i,ext in enumerate((".tar.gz", ".tgz", ".tar.bz2")):
+            path = pjoin(self.src, "myjsext" + ext)
+            with tarfile.open(path, 'w') as f:
+                _add_file(f, "b%i.js" % i, b"b();")
+                _add_file(f, "foo/b%i.js" % i, b"foo();")
+            install_nbextension(path)
+            self.assert_installed("b%i.js" % i)
+            self.assert_installed(pjoin("foo", "b%i.js" % i))
+    
+    def test_install_url(self):
+        def fake_urlretrieve(url, dest):
+            touch(dest)
+        save_urlretrieve = nbextensions.urlretrieve
+        nbextensions.urlretrieve = fake_urlretrieve
+        try:
+            install_nbextension("http://example.com/path/to/foo.js")
+            self.assert_installed("foo.js")
+            install_nbextension("https://example.com/path/to/another/bar.js")
+            self.assert_installed("bar.js")
+        finally:
+            nbextensions.urlretrieve = save_urlretrieve
+    
+    def test_check_nbextension(self):
+        with TemporaryDirectory() as d:
+            f = u'ƒ.js'
+            src = pjoin(d, f)
+            touch(src)
+            install_nbextension(src)
+        
+        assert check_nbextension(f, self.ipdir)
+        assert check_nbextension([f], self.ipdir)
+        assert not check_nbextension([f, pjoin('dne', f)], self.ipdir)
+    
+    @dec.skip_win32
+    def test_install_symlink(self):
+        with TemporaryDirectory() as d:
+            f = u'ƒ.js'
+            src = pjoin(d, f)
+            touch(src)
+            install_nbextension(src, symlink=True)
+        dest = pjoin(self.ipdir, u'nbextensions', f)
+        assert os.path.islink(dest)
+        link = os.readlink(dest)
+        self.assertEqual(link, src)
+    
+    def test_install_symlink_bad(self):
+        with self.assertRaises(ValueError):
+            install_nbextension("http://example.com/foo.js", symlink=True)
+        
+        with TemporaryDirectory() as d:
+            zf = u'ƒ.zip'
+            zsrc = pjoin(d, zf)
+            with zipfile.ZipFile(zsrc, 'w') as z:
+                z.writestr("a.js", b"b();")
+        
+            with self.assertRaises(ValueError):
+                install_nbextension(zsrc, symlink=True)
+

IPython/html/__init__.py

@@ -5,3 +5,5 b' import os'
 DEFAULT_STATIC_FILES_PATH = os.path.join(os.path.dirname(__file__), "static")
 
 del os
+
+from .nbextensions import install_nbextension
  No newline at end of file

IPython/html/static/base/js/utils.js

@@ -12,6 +12,34 b" IPython.namespace('IPython.utils');"
 
 IPython.utils = (function (IPython) {
     "use strict";
+    
+    IPython.load_extensions = function () {
+        // load one or more IPython notebook extensions with requirejs
+        
+        var extensions = [];
+        var extension_names = arguments;
+        for (var i = 0; i < extension_names.length; i++) {
+            extensions.push("nbextensions/" + arguments[i]);
+        }
+        
+        require(extensions,
+            function () {
+                for (var i = 0; i < arguments.length; i++) {
+                    var ext = arguments[i];
+                    var ext_name = extension_names[i];
+                    // success callback
+                    console.log("Loaded extension: " + ext_name);
+                    if (ext && ext.load_ipython_extension !== undefined) {
+                        ext.load_ipython_extension();
+                    }
+                }
+            },
+            function (err) {
+                // failure callback
+                console.log("Failed to load extension(s):", err.requireModules, err);
+            }
+        );
+    };
 
     //============================================================================
     // Cross-browser RegEx Split

IPython/terminal/ipapp.py

@@ -224,7 +224,7 b' class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):'
             StoreMagics,
         ]
 
-    subcommands = Dict(dict(
+    subcommands = dict(
         qtconsole=('IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp',
             """Launch the IPython Qt Console."""
         ),
@@ -252,7 +252,11 b' class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp):'
         trust=('IPython.nbformat.sign.TrustNotebookApp',
             "Sign notebooks to trust their potentially unsafe contents at load."
         ),
-    ))
+    )
+    subcommands['install-nbextension'] = (
+        "IPython.html.nbextensions.NBExtensionApp",
+        "Install IPython notebook extension files"
+    )
 
     # *do* autocreate requested profile, but don't create the config file.
     auto_create=Bool(True)

IPython/testing/tools.py

@@ -327,6 +327,8 b' else:'
             s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING)
             super(MyStringIO, self).write(s)
 
+_re_type = type(re.compile(r''))
+
 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
 -------
 {2!s}
@@ -347,7 +349,7 b' class AssertPrints(object):'
     """
     def __init__(self, s, channel='stdout', suppress=True):
         self.s = s
-        if isinstance(self.s, py3compat.string_types):
+        if isinstance(self.s, (py3compat.string_types, _re_type)):
             self.s = [self.s]
         self.channel = channel
         self.suppress = suppress
@@ -366,7 +368,10 b' class AssertPrints(object):'
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         for s in self.s:
-            assert s in printed, notprinted_msg.format(s, self.channel, printed)
+            if isinstance(s, _re_type):
+                assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
+            else:
+                assert s in printed, notprinted_msg.format(s, self.channel, printed)
         return False
 
 printed_msg = """Found {0!r} in printed output (on {1}):
@@ -387,7 +392,10 b' class AssertNotPrints(AssertPrints):'
         setattr(sys, self.channel, self.orig_stream)
         printed = self.buffer.getvalue()
         for s in self.s:
-            assert s not in printed, printed_msg.format(s, self.channel, printed)
+            if isinstance(s, _re_type):
+                assert not s.search(printed), printed_msg.format(s.pattern, self.channel, printed)
+            else:
+                assert s not in printed, printed_msg.format(s, self.channel, printed)
         return False
 
 @contextmanager