support URLs and zip/tarballs in install_extension

IPython/html/nbextensions.py

@@ -1,176 +1,224 b''
 # coding: utf-8
 """Utilities for installing Javascript extensions for the notebook"""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2014 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 import os
 import shutil
+import tarfile
+import zipfile
 from os.path import basename, join as pjoin
 
+# Deferred imports
+try:
+    from urllib.parse import urlparse  # Py3
+    from urllib.request import urlretrieve
+except ImportError:
+    from urlparse import urlparse
+    from urllib import urlretrieve
+
 from IPython.utils.path import get_ipython_dir
 from IPython.utils.py3compat import string_types, cast_unicode_py2
+from IPython.utils.tempdir import TemporaryDirectory
 
 
 def _should_copy(src, dest, verbose=1):
     """should a file be copied?"""
     if not os.path.exists(dest):
         return True
     if os.stat(dest).st_mtime < os.stat(src).st_mtime:
         if verbose >= 2:
             print("%s is out of date" % dest)
         return True
     if verbose >= 2:
         print("%s is up to date" % dest)
     return False
 
 
 def _maybe_copy(src, dest, verbose=1):
     """copy a file if it needs updating"""
     if _should_copy(src, dest, verbose):
         if verbose >= 1:
             print("copying %s -> %s" % (src, dest))
         shutil.copy2(src, dest)
 
 
+def _safe_is_tarfile(path):
+    """safe version of is_tarfile, return False on IOError"""
+    try:
+        return tarfile.is_tarfile(path)
+    except IOError:
+        return False
+
+
 def install_nbextension(files, overwrite=False, ipython_dir=None, verbose=1):
     """Install a Javascript extension for the notebook
     
     Stages files and/or directories into IPYTHONDIR/nbextensions.
-    By default, this comparse modification time, and only stages files that need updating.
+    By default, this compares modification time, and only stages files that need updating.
     If `overwrite` is specified, matching files are purged before proceeding.
     
     Parameters
     ----------
     
-    files : list(paths)
-        One or more paths to existing files or directories to install.
+    files : list(paths or URLs)
+        One or more paths or URLs to existing files directories to install.
         These will be installed with their base name, so '/path/to/foo'
         will install to 'nbextensions/foo'.
+        Archives (zip or tarballs) will be extracted into the nbextensions directory.
     overwrite : bool [default: False]
         If True, always install the files, regardless of what may already be installed.
     ipython_dir : str [optional]
         The path to an IPython directory, if the default value is not desired.
         get_ipython_dir() is used by default.
     verbose : int [default: 1]
         Set verbosity level. The default is 1, where file actions are printed.
         set verbose=2 for more output, or verbose=0 for silence.
     """
     
     ipython_dir = ipython_dir or get_ipython_dir()
     nbext = pjoin(ipython_dir, u'nbextensions')
     # make sure nbextensions dir exists
     if not os.path.exists(nbext):
         os.makedirs(nbext)
     
     if isinstance(files, string_types):
         # one file given, turn it into a list
         files = [files]
     
     for path in map(cast_unicode_py2, files):
+        
+        if path.startswith(('https://', 'http://')):
+            # Given a URL, download it
+            with TemporaryDirectory() as td:
+                filename = urlparse(path).path.split('/')[-1]
+                local_path = os.path.join(td, filename)
+                if verbose >= 1:
+                    print("downloading %s to %s" % (path, local_path))
+                urlretrieve(path, local_path)
+                # now install from the local copy
+                install_nbextension(local_path, overwrite, ipython_dir, verbose)
+            continue
+        
+        # handle archives
+        archive = None
+        if path.endswith('.zip'):
+            archive = zipfile.ZipFile(path)
+        elif _safe_is_tarfile(path):
+            archive = tarfile.open(path)
+        
+        if archive:
+            if verbose >= 1:
+                print("extracting %s to %s" % (path, nbext))
+            archive.extractall(nbext)
+            archive.close()
+            continue
+        
         dest = pjoin(nbext, basename(path))
         if overwrite and os.path.exists(dest):
             if verbose >= 1:
                 print("removing %s" % dest)
             if os.path.isdir(dest):
                 shutil.rmtree(dest)
             else:
                 os.remove(dest)
 
         if os.path.isdir(path):
             strip_prefix_len = len(path) - len(basename(path))
             for parent, dirs, files in os.walk(path):
                 dest_dir = pjoin(nbext, parent[strip_prefix_len:])
                 if not os.path.exists(dest_dir):
                     if verbose >= 2:
                         print("making directory %s" % dest_dir)
                     os.makedirs(dest_dir)
                 for file in files:
                     src = pjoin(parent, file)
                     # print("%r, %r" % (dest_dir, file))
                     dest = pjoin(dest_dir, file)
                     _maybe_copy(src, dest, verbose)
         else:
             src = path
             _maybe_copy(src, dest, verbose)
 
 #----------------------------------------------------------------------
 # install nbextension app
 #----------------------------------------------------------------------
 
-import logging
 from IPython.utils.traitlets import Bool, Enum
 from IPython.core.application import BaseIPythonApplication
 
 flags = {
     "overwrite" : ({
         "NBExtensionApp" : {
             "overwrite" : True,
         }}, "Force overwrite of existing files"
     ),
     "debug" : ({
         "NBExtensionApp" : {
             "verbose" : 2,
         }}, "Extra output"
     ),
     "quiet" : ({
         "NBExtensionApp" : {
             "verbose" : 0,
         }}, "Minimal output"
     ),
 }
 aliases = {
     "ipython-dir" : "NBExtensionApp.ipython_dir"
 }
 
 class NBExtensionApp(BaseIPythonApplication):
     """Entry point for installing notebook extensions"""
     
     description = """Install IPython notebook extensions
     
     Usage
     
-        ipython install-nbextension file [more files or folders]
+        ipython install-nbextension file [more files, folders, archives or urls]
     
     This copies files and/or folders into the IPython nbextensions directory.
+    If a URL is given, it will be downloaded.
+    If an archive is given, it will be extracted into nbextensions.
     If the requested files are already up to date, no action is taken
     unless --overwrite is specified.
     """
     
     examples = """
     ipython install-nbextension /path/to/d3.js /path/to/myextension
     """
     aliases = aliases
     flags = flags
     
     overwrite = Bool(False, config=True, help="Force overwrite of existing files")
     verbose = Enum((0,1,2), default_value=1, config=True,
         help="Verbosity level"
     )
     
     def install_extensions(self):
         install_nbextension(self.extra_args,
             overwrite=self.overwrite,
             verbose=self.verbose,
             ipython_dir=self.ipython_dir,
         )
     
     def start(self):
         if not self.extra_args:
             nbext = pjoin(self.ipython_dir, u'nbextensions')
             print("Notebook extensions in %s:" % nbext)
             for ext in os.listdir(nbext):
                 print(u"    %s" % ext)
         else:
             self.install_extensions()
 
 
 if __name__ == '__main__':
     NBExtensionApp.launch_instance()
     
  No newline at end of file

IPython/html/tests/test_nbextensions.py

@@ -1,201 +1,234 b''
 # coding: utf-8
 """Test installation of notebook extensions"""
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2014 The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import glob
 import os
 import re
-import time
-from contextlib import contextmanager
+import tarfile
+import zipfile
+from io import BytesIO
 from os.path import basename, join as pjoin
 from unittest import TestCase
 
-import nose.tools as nt
-
-from IPython.external.decorator import decorator
-
 import IPython.testing.tools as tt
-import IPython.utils.path
 from IPython.utils import py3compat
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.html import nbextensions
 from IPython.html.nbextensions import install_nbextension
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 def touch(file, mtime=None):
     """ensure a file exists, and set its modification time
     
     returns the modification time of the file
     """
     open(file, 'a').close()
     # set explicit mtime
     if mtime:
         atime = os.stat(file).st_atime
         os.utime(file, (atime, mtime))
     return os.stat(file).st_mtime
 
 
 class TestInstallNBExtension(TestCase):
     
     def tempdir(self):
         td = TemporaryDirectory()
         self.tempdirs.append(td)
         return py3compat.cast_unicode(td.name)
 
     def setUp(self):
         self.tempdirs = []
         src = self.src = self.tempdir()
         self.files = files = [
             pjoin(u'ƒile'),
             pjoin(u'∂ir', u'ƒile1'),
             pjoin(u'∂ir', u'∂ir2', u'ƒile2'),
         ]
         for file in files:
             fullpath = os.path.join(self.src, file)
             parent = os.path.dirname(fullpath)
             if not os.path.exists(parent):
                 os.makedirs(parent)
             touch(fullpath)
         
         self.ipdir = self.tempdir()
         self.save_get_ipython_dir = nbextensions.get_ipython_dir
         nbextensions.get_ipython_dir = lambda : self.ipdir
     
     def tearDown(self):
         for td in self.tempdirs:
             td.cleanup()
         nbextensions.get_ipython_dir = self.save_get_ipython_dir
     
     def assert_path_exists(self, path):
         if not os.path.exists(path):
-            self.fail(u"%s should exist" % path)
+            do_exist = os.listdir(os.path.dirname(path))
+            self.fail(u"%s should exist (found %s)" % (path, do_exist))
     
     def assert_not_path_exists(self, path):
         if os.path.exists(path):
             self.fail(u"%s should not exist" % path)
     
     def assert_installed(self, relative_path, ipdir=None):
         self.assert_path_exists(
             pjoin(ipdir or self.ipdir, u'nbextensions', relative_path)
         )
     
     def assert_not_installed(self, relative_path, ipdir=None):
         self.assert_not_path_exists(
             pjoin(ipdir or self.ipdir, u'nbextensions', relative_path)
         )
     
     def test_create_ipython_dir(self):
         """install_nbextension when ipython_dir doesn't exist"""
         with TemporaryDirectory() as td:
             ipdir = pjoin(td, u'ipython')
             install_nbextension(self.src, ipython_dir=ipdir)
             self.assert_path_exists(ipdir)
             for file in self.files:
                 self.assert_installed(
                     pjoin(basename(self.src), file),
                     ipdir
                 )
     
     def test_create_nbextensions(self):
         with TemporaryDirectory() as ipdir:
             install_nbextension(self.src, ipython_dir=ipdir)
             self.assert_installed(
                 pjoin(basename(self.src), u'ƒile'),
                 ipdir
             )
     
     def test_single_file(self):
         file = self.files[0]
         install_nbextension(pjoin(self.src, file))
         self.assert_installed(file)
     
     def test_single_dir(self):
         d = u'∂ir'
         install_nbextension(pjoin(self.src, d))
         self.assert_installed(self.files[-1])
     
     def test_install_nbextension(self):
         install_nbextension(glob.glob(pjoin(self.src, '*')))
         for file in self.files:
             self.assert_installed(file)
     
     def test_overwrite_file(self):
         with TemporaryDirectory() as d:
             fname = u'ƒ.js'
             src = pjoin(d, fname)
             with open(src, 'w') as f:
                 f.write('first')
             mtime = touch(src)
             dest = pjoin(self.ipdir, u'nbextensions', fname)
             install_nbextension(src)
             with open(src, 'w') as f:
                 f.write('overwrite')
             mtime = touch(src, mtime - 100)
             install_nbextension(src, overwrite=True)
             with open(dest) as f:
                 self.assertEqual(f.read(), 'overwrite')
     
     def test_overwrite_dir(self):
         with TemporaryDirectory() as src:
             # src = py3compat.cast_unicode_py2(src)
             base = basename(src)
             fname = u'ƒ.js'
             touch(pjoin(src, fname))
             install_nbextension(src)
             self.assert_installed(pjoin(base, fname))
             os.remove(pjoin(src, fname))
             fname2 = u'∂.js'
             touch(pjoin(src, fname2))
             install_nbextension(src, overwrite=True)
             self.assert_installed(pjoin(base, fname2))
             self.assert_not_installed(pjoin(base, fname))
     
     def test_update_file(self):
         with TemporaryDirectory() as d:
             fname = u'ƒ.js'
             src = pjoin(d, fname)
             with open(src, 'w') as f:
                 f.write('first')
             mtime = touch(src)
             install_nbextension(src)
             self.assert_installed(fname)
             dest = pjoin(self.ipdir, u'nbextensions', fname)
             old_mtime = os.stat(dest).st_mtime
             with open(src, 'w') as f:
                 f.write('overwrite')
             touch(src, mtime + 10)
             install_nbextension(src)
             with open(dest) as f:
                 self.assertEqual(f.read(), 'overwrite')
     
     def test_skip_old_file(self):
         with TemporaryDirectory() as d:
             fname = u'ƒ.js'
             src = pjoin(d, fname)
             mtime = touch(src)
             install_nbextension(src)
             self.assert_installed(fname)
             dest = pjoin(self.ipdir, u'nbextensions', fname)
             old_mtime = os.stat(dest).st_mtime
             
             mtime = touch(src, mtime - 100)
             install_nbextension(src)
             new_mtime = os.stat(dest).st_mtime
             self.assertEqual(new_mtime, old_mtime)
 
     def test_quiet(self):
         with tt.AssertNotPrints(re.compile(r'.+')):
             install_nbextension(self.src, verbose=0)
     
+    def test_install_zip(self):
+        path = pjoin(self.src, "myjsext.zip")
+        with zipfile.ZipFile(path, 'w') as f:
+            f.writestr("a.js", b"b();")
+            f.writestr("foo/a.js", b"foo();")
+        install_nbextension(path)
+        self.assert_installed("a.js")
+        self.assert_installed(pjoin("foo", "a.js"))
+    
+    def test_install_tar(self):
+        def _add_file(f, fname, buf):
+            info = tarfile.TarInfo(fname)
+            info.size = len(buf)
+            f.addfile(info, BytesIO(buf))
+        
+        for i,ext in enumerate((".tar.gz", ".tgz", ".tar.bz2")):
+            path = pjoin(self.src, "myjsext" + ext)
+            with tarfile.open(path, 'w') as f:
+                _add_file(f, "b%i.js" % i, b"b();")
+                _add_file(f, "foo/b%i.js" % i, b"foo();")
+            install_nbextension(path)
+            self.assert_installed("b%i.js" % i)
+            self.assert_installed(pjoin("foo", "b%i.js" % i))
+    
+    def test_install_url(self):
+        def fake_urlretrieve(url, dest):
+            touch(dest)
+        save_urlretrieve = nbextensions.urlretrieve
+        nbextensions.urlretrieve = fake_urlretrieve
+        try:
+            install_nbextension("http://example.com/path/to/foo.js")
+            self.assert_installed("foo.js")
+            install_nbextension("https://example.com/path/to/another/bar.js")
+            self.assert_installed("bar.js")
+        finally:
+            nbextensions.urlretrieve = save_urlretrieve