Refactoring notebook managers and adding Azure backed storage.

I have created a base class for all notebook managers. Our
existing, file-based store, is now in filenbmanager.py. I have
also created a new Azure Blob based backed notebook manager.

IPython/frontend/html/notebook/azurenbmanager.py

@@ -0,0 +1,140 b''
+"""A notebook manager that uses Azure blob storage.
+
+Authors:
+
+* Brian Granger
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2012  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import datetime
+
+import azure
+from azure.storage import BlobService
+
+from tornado import web
+
+from .basenbmanager import BaseNotebookManager
+from IPython.nbformat import current
+from IPython.utils.traitlets import Unicode, Instance
+
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+class AzureNotebookManager(BaseNotebookManager):
+
+    account_name = Unicode('', config=True, help='Azure storage account name.')
+    account_key = Unicode('', config=True, help='Azure storage account key.')
+    container = Unicode('', config=True, help='Container name for notebooks.')
+
+    blob_service_host_base = Unicode('.blob.core.windows.net', config=True,
+        help='The basename for the blob service URL. If running on the preview site this '
+             'will be .blob.core.azure-preview.com.')
+    def _blob_service_host_base_changed(self, new):
+        self._update_service_host_base(new)
+
+    blob_service = Instance('azure.storage.BlobService')
+    def _blob_service_default(self):
+        return BlobService(account_name=self.account_name, account_key=self.account_key)
+
+    def __init__(self, **kwargs):
+        super(BaseNotebookManager,self).__init__(**kwargs)
+        self._update_service_host_base(self.blob_service_host_base)
+        self._create_container()
+
+    def _update_service_host_base(self, shb):
+        azure.BLOB_SERVICE_HOST_BASE = shb
+
+    def _create_container(self):
+        self.blob_service.create_container(self.container)
+
+    def load_notebook_names(self):
+        """On startup load the notebook ids and names from Azure.
+
+        The blob names are the notebook ids and the notebook names are stored
+        as blob metadata.
+        """
+        self.mapping = {}
+        blobs = self.blob_service.list_blobs(self.container)
+        ids = [blob.name for blob in blobs]
+        
+        for id in ids:
+            md = self.blob_service.get_blob_metadata(self.container, id)
+            name = md['x-ms-meta-nbname']
+            self.mapping[id] = name
+
+    def list_notebooks(self):
+        """List all notebooks in the container.
+
+        This version uses `self.mapping` as the authoritative notebook list.
+        """
+        data = [dict(notebook_id=id,name=name) for id, name in self.mapping.items()]
+        data = sorted(data, key=lambda item: item['name'])
+        return data
+
+    def read_notebook_object(self, notebook_id):
+        """Get the object representation of a notebook by notebook_id."""
+        if not self.notebook_exists(notebook_id):
+            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
+        try:
+            s = self.blob_service.get_blob(self.container, notebook_id)
+        except:
+            raise web.HTTPError(500, u'Notebook cannot be read.')
+        try:
+            # v1 and v2 and json in the .ipynb files.
+            nb = current.reads(s, u'json')
+        except:
+            raise web.HTTPError(500, u'Unreadable JSON notebook.')
+        # Todo: The last modified should actually be saved in the notebook document.
+        # We are just using the current datetime until that is implemented.
+        last_modified = datetime.datetime.utcnow()
+        return last_modified, nb
+
+    def write_notebook_object(self, nb, notebook_id=None):
+        """Save an existing notebook object by notebook_id."""
+        try:
+            new_name = nb.metadata.name
+        except AttributeError:
+            raise web.HTTPError(400, u'Missing notebook name')
+
+        if notebook_id is None:
+            notebook_id = self.new_notebook_id(new_name)
+
+        if notebook_id not in self.mapping:
+            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
+
+        try:
+            data = current.writes(nb, u'json')
+        except Exception as e:
+            raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)
+
+        metadata = {'nbname': new_name}
+        try:
+            self.blob_service.put_blob(self.container, notebook_id, data, 'BlockBlob', x_ms_meta_name_values=metadata)
+        except Exception as e:
+            raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)
+
+        self.mapping[notebook_id] = new_name
+        return notebook_id
+
+    def delete_notebook(self, notebook_id):
+        """Delete notebook by notebook_id."""
+        if not self.notebook_exists(notebook_id):
+            raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
+        try:
+            self.blob_service.delete_blob(self.container, notebook_id)
+        except Exception as e:
+            raise web.HTTPError(400, u'Unexpected error while deleting notebook: %s' % e)
+        else:
+            self.delete_notebook_id(notebook_id)

IPython/frontend/html/notebook/basenbmanager.py

@@ -0,0 +1,181 b''
+"""A base class notebook manager.
+
+Authors:
+
+* Brian Granger
+"""
+
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2011  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+import uuid
+
+from tornado import web
+
+from IPython.config.configurable import LoggingConfigurable
+from IPython.nbformat import current
+from IPython.utils.traitlets import List, Dict
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+class BaseNotebookManager(LoggingConfigurable):
+
+    allowed_formats = List([u'json',u'py'])
+
+    # Map notebook_ids to notebook names
+    mapping = Dict()
+
+    def load_notebook_names(self):
+        """Load the notebook names into memory.
+
+        This should be called once immediately after the notebook manager
+        is created to load the existing notebooks into the mapping in
+        memory.
+        """
+        self.list_notebooks()
+
+    def list_notebooks(self):
+        """List all notebooks.
+
+        This returns a list of dicts, each of the form::
+
+            dict(notebook_id=notebook,name=name)
+
+        This list of dicts should be sorted by name::
+
+            data = sorted(data, key=lambda item: item['name'])
+        """
+        raise NotImplementedError('must be implemented in a subclass')
+
+
+    def new_notebook_id(self, name):
+        """Generate a new notebook_id for a name and store its mapping."""
+        # TODO: the following will give stable urls for notebooks, but unless
+        # the notebooks are immediately redirected to their new urls when their
+        # filemname changes, nasty inconsistencies result.  So for now it's
+        # disabled and instead we use a random uuid4() call.  But we leave the
+        # logic here so that we can later reactivate it, whhen the necessary
+        # url redirection code is written.
+        #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,
+        #                 'file://'+self.get_path_by_name(name).encode('utf-8')))
+        
+        notebook_id = unicode(uuid.uuid4())
+        self.mapping[notebook_id] = name
+        return notebook_id
+
+    def delete_notebook_id(self, notebook_id):
+        """Delete a notebook's id in the mapping.
+
+        This doesn't delete the actual notebook, only its entry in the mapping.
+        """
+        del self.mapping[notebook_id]
+
+    def notebook_exists(self, notebook_id):
+        """Does a notebook exist?"""
+        return notebook_id in self.mapping
+
+    def get_notebook(self, notebook_id, format=u'json'):
+        """Get the representation of a notebook in format by notebook_id."""
+        format = unicode(format)
+        if format not in self.allowed_formats:
+            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
+        last_modified, nb = self.read_notebook_object(notebook_id)
+        kwargs = {}
+        if format == 'json':
+            # don't split lines for sending over the wire, because it
+            # should match the Python in-memory format.
+            kwargs['split_lines'] = False
+        data = current.writes(nb, format, **kwargs)
+        name = nb.get('name','notebook')
+        return last_modified, name, data
+
+    def read_notebook_object(self, notebook_id):
+        """Get the object representation of a notebook by notebook_id."""
+        raise NotImplementedError('must be implemented in a subclass')
+
+    def save_new_notebook(self, data, name=None, format=u'json'):
+        """Save a new notebook and return its notebook_id.
+
+        If a name is passed in, it overrides any values in the notebook data
+        and the value in the data is updated to use that value.
+        """
+        if format not in self.allowed_formats:
+            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
+
+        try:
+            nb = current.reads(data.decode('utf-8'), format)
+        except:
+            raise web.HTTPError(400, u'Invalid JSON data')
+
+        if name is None:
+            try:
+                name = nb.metadata.name
+            except AttributeError:
+                raise web.HTTPError(400, u'Missing notebook name')
+        nb.metadata.name = name
+
+        notebook_id = self.write_notebook_object(nb)
+        return notebook_id
+
+    def save_notebook(self, notebook_id, data, name=None, format=u'json'):
+        """Save an existing notebook by notebook_id."""
+        if format not in self.allowed_formats:
+            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
+
+        try:
+            nb = current.reads(data.decode('utf-8'), format)
+        except:
+            raise web.HTTPError(400, u'Invalid JSON data')
+
+        if name is not None:
+            nb.metadata.name = name
+        self.write_notebook_object(nb, notebook_id)
+
+    def write_notebook_object(self, nb, notebook_id=None):
+        """Write a notebook object and return its notebook_id.
+
+        If notebook_id is None, this method should create a new notebook_id.
+        If notebook_id is not None, this method should check to make sure it
+        exists and is valid.
+        """
+        raise NotImplementedError('must be implemented in a subclass')
+
+    def delete_notebook(self, notebook_id):
+        """Delete notebook by notebook_id."""
+        raise NotImplementedError('must be implemented in a subclass')
+
+    def increment_filename(self, name):
+        """Increment a filename to make it unique.
+
+        This exists for notebook stores that must have unique names. When a notebook
+        is created or copied this method constructs a unique filename, typically
+        by appending an integer to the name.
+        """
+        return name
+
+    def new_notebook(self):
+        """Create a new notebook and return its notebook_id."""
+        name = self.increment_filename('Untitled')
+        metadata = current.new_metadata(name=name)
+        nb = current.new_notebook(metadata=metadata)
+        notebook_id = self.write_notebook_object(nb)
+        return notebook_id
+
+    def copy_notebook(self, notebook_id):
+        """Copy an existing notebook and return its notebook_id."""
+        last_mod, nb = self.read_notebook_object(notebook_id)
+        name = nb.metadata.name + '-Copy'
+        name = self.increment_filename(name)
+        nb.metadata.name = name
+        notebook_id = self.write_notebook_object(nb)
+        return notebook_id

IPython/frontend/html/notebook/filenbmanager.py

@@ -1,286 +1,207 b''
 """A notebook manager that uses the local file system for storage.
 
 Authors:
 
 * Brian Granger
 """
 
 #-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
+#  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import datetime
 import io
 import os
-import uuid
 import glob
 
 from tornado import web
 
-from IPython.config.configurable import LoggingConfigurable
+from .basenbmanager import BaseNotebookManager
 from IPython.nbformat import current
-from IPython.utils.traitlets import Unicode, List, Dict, Bool, TraitError
+from IPython.utils.traitlets import Unicode, Dict, Bool, TraitError
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
-class NotebookManager(LoggingConfigurable):
+class FileNotebookManager(BaseNotebookManager):
 
     notebook_dir = Unicode(os.getcwdu(), config=True, help="""
         The directory to use for notebooks.
     """)
     def _notebook_dir_changed(self, name, old, new):
         """do a bit of validation of the notebook dir"""
         if os.path.exists(new) and not os.path.isdir(new):
             raise TraitError("notebook dir %r is not a directory" % new)
         if not os.path.exists(new):
             self.log.info("Creating notebook dir %s", new)
             try:
                 os.mkdir(new)
             except:
                 raise TraitError("Couldn't create notebook dir %r" % new)
     
     save_script = Bool(False, config=True,
         help="""Automatically create a Python script when saving the notebook.
         
         For easier use of import, %run and %load across notebooks, a
         <notebook-name>.py script will be created next to any
         <notebook-name>.ipynb on each save.  This can also be set with the
         short `--script` flag.
         """
     )
     
     filename_ext = Unicode(u'.ipynb')
-    allowed_formats = List([u'json',u'py'])
 
-    # Map notebook_ids to notebook names
-    mapping = Dict()
     # Map notebook names to notebook_ids
     rev_mapping = Dict()
 
-    def list_notebooks(self):
-        """List all notebooks in the notebook dir.
-
-        This returns a list of dicts of the form::
-
-            dict(notebook_id=notebook,name=name)
-        """
+    def get_notebook_names(self):
+        """List all notebook names in the notebook dir."""
         names = glob.glob(os.path.join(self.notebook_dir,
                                        '*' + self.filename_ext))
         names = [os.path.splitext(os.path.basename(name))[0]
                  for name in names]
+        return names
+
+    def list_notebooks(self):
+        """List all notebooks in the notebook dir."""
+        names = self.get_notebook_names()
 
         data = []
         for name in names:
             if name not in self.rev_mapping:
                 notebook_id = self.new_notebook_id(name)
             else:
                 notebook_id = self.rev_mapping[name]
             data.append(dict(notebook_id=notebook_id,name=name))
         data = sorted(data, key=lambda item: item['name'])
         return data
 
     def new_notebook_id(self, name):
         """Generate a new notebook_id for a name and store its mappings."""
-        # TODO: the following will give stable urls for notebooks, but unless
-        # the notebooks are immediately redirected to their new urls when their
-        # filemname changes, nasty inconsistencies result.  So for now it's
-        # disabled and instead we use a random uuid4() call.  But we leave the
-        # logic here so that we can later reactivate it, whhen the necessary
-        # url redirection code is written.
-        #notebook_id = unicode(uuid.uuid5(uuid.NAMESPACE_URL,
-        #                 'file://'+self.get_path_by_name(name).encode('utf-8')))
-        
-        notebook_id = unicode(uuid.uuid4())
-        
-        self.mapping[notebook_id] = name
+        notebook_id = super(BaseNotebookManager, self).new_notebook_id(name)
         self.rev_mapping[name] = notebook_id
         return notebook_id
 
     def delete_notebook_id(self, notebook_id):
-        """Delete a notebook's id only. This doesn't delete the actual notebook."""
+        """Delete a notebook's id in the mapping."""
+        super(BaseNotebookManager, self).delete_notebook_id(notebook_id)
         name = self.mapping[notebook_id]
-        del self.mapping[notebook_id]
         del self.rev_mapping[name]
 
     def notebook_exists(self, notebook_id):
         """Does a notebook exist?"""
-        if notebook_id not in self.mapping:
+        exists = super(BaseNotebookManager, self).notebook_exists(notebook_id)
+        if not exists:
             return False
         path = self.get_path_by_name(self.mapping[notebook_id])
         return os.path.isfile(path)
 
     def find_path(self, notebook_id):
         """Return a full path to a notebook given its notebook_id."""
         try:
             name = self.mapping[notebook_id]
         except KeyError:
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         return self.get_path_by_name(name)
 
     def get_path_by_name(self, name):
         """Return a full path to a notebook given its name."""
         filename = name + self.filename_ext
         path = os.path.join(self.notebook_dir, filename)
         return path       
 
-    def get_notebook(self, notebook_id, format=u'json'):
-        """Get the representation of a notebook in format by notebook_id."""
-        format = unicode(format)
-        if format not in self.allowed_formats:
-            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
-        last_modified, nb = self.get_notebook_object(notebook_id)
-        kwargs = {}
-        if format == 'json':
-            # don't split lines for sending over the wire, because it
-            # should match the Python in-memory format.
-            kwargs['split_lines'] = False
-        data = current.writes(nb, format, **kwargs)
-        name = nb.get('name','notebook')
-        return last_modified, name, data
-
-    def get_notebook_object(self, notebook_id):
+    def read_notebook_object(self, notebook_id):
         """Get the NotebookNode representation of a notebook by notebook_id."""
         path = self.find_path(notebook_id)
         if not os.path.isfile(path):
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         info = os.stat(path)
         last_modified = datetime.datetime.utcfromtimestamp(info.st_mtime)
         with open(path,'r') as f:
             s = f.read()
             try:
                 # v1 and v2 and json in the .ipynb files.
                 nb = current.reads(s, u'json')
             except:
                 raise web.HTTPError(500, u'Unreadable JSON notebook.')
         # Always use the filename as the notebook name.
         nb.metadata.name = os.path.splitext(os.path.basename(path))[0]
         return last_modified, nb
 
-    def save_new_notebook(self, data, name=None, format=u'json'):
-        """Save a new notebook and return its notebook_id.
-
-        If a name is passed in, it overrides any values in the notebook data
-        and the value in the data is updated to use that value.
-        """
-        if format not in self.allowed_formats:
-            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
-
-        try:
-            nb = current.reads(data.decode('utf-8'), format)
-        except:
-            raise web.HTTPError(400, u'Invalid JSON data')
-
-        if name is None:
-            try:
-                name = nb.metadata.name
-            except AttributeError:
-                raise web.HTTPError(400, u'Missing notebook name')
-        nb.metadata.name = name
-
-        notebook_id = self.new_notebook_id(name)
-        self.save_notebook_object(notebook_id, nb)
-        return notebook_id
-
-    def save_notebook(self, notebook_id, data, name=None, format=u'json'):
-        """Save an existing notebook by notebook_id."""
-        if format not in self.allowed_formats:
-            raise web.HTTPError(415, u'Invalid notebook format: %s' % format)
-
+    def write_notebook_object(self, nb, notebook_id=None):
+        """Save an existing notebook object by notebook_id."""
         try:
-            nb = current.reads(data.decode('utf-8'), format)
-        except:
-            raise web.HTTPError(400, u'Invalid JSON data')
+            new_name = nb.metadata.name
+        except AttributeError:
+            raise web.HTTPError(400, u'Missing notebook name')
 
-        if name is not None:
-            nb.metadata.name = name
-        self.save_notebook_object(notebook_id, nb)
+        if notebook_id is None:
+            notebook_id = self.new_notebook_id(new_name)
 
-    def save_notebook_object(self, notebook_id, nb):
-        """Save an existing notebook object by notebook_id."""
         if notebook_id not in self.mapping:
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
+
         old_name = self.mapping[notebook_id]
-        try:
-            new_name = nb.metadata.name
-        except AttributeError:
-            raise web.HTTPError(400, u'Missing notebook name')
         path = self.get_path_by_name(new_name)
         try:
             with open(path,'w') as f:
                 current.write(nb, f, u'json')
         except Exception as e:
             raise web.HTTPError(400, u'Unexpected error while saving notebook: %s' % e)
+
         # save .py script as well
         if self.save_script:
             pypath = os.path.splitext(path)[0] + '.py'
             try:
                 with io.open(pypath,'w', encoding='utf-8') as f:
                     current.write(nb, f, u'py')
             except Exception as e:
                 raise web.HTTPError(400, u'Unexpected error while saving notebook as script: %s' % e)
         
+        # remove old files if the name changed
         if old_name != new_name:
             old_path = self.get_path_by_name(old_name)
             if os.path.isfile(old_path):
                 os.unlink(old_path)
             if self.save_script:
                 old_pypath = os.path.splitext(old_path)[0] + '.py'
                 if os.path.isfile(old_pypath):
                     os.unlink(old_pypath)
             self.mapping[notebook_id] = new_name
             self.rev_mapping[new_name] = notebook_id
             del self.rev_mapping[old_name]
+        
+        return notebook_id
 
     def delete_notebook(self, notebook_id):
         """Delete notebook by notebook_id."""
         path = self.find_path(notebook_id)
         if not os.path.isfile(path):
             raise web.HTTPError(404, u'Notebook does not exist: %s' % notebook_id)
         os.unlink(path)
         self.delete_notebook_id(notebook_id)
 
     def increment_filename(self, basename):
         """Return a non-used filename of the form basename<int>.
         
         This searches through the filenames (basename0, basename1, ...)
         until is find one that is not already being used. It is used to
         create Untitled and Copy names that are unique.
         """
         i = 0
         while True:
             name = u'%s%i' % (basename,i)
             path = self.get_path_by_name(name)
             if not os.path.isfile(path):
                 break
             else:
                 i = i+1
-        return path, name
-
-    def new_notebook(self):
-        """Create a new notebook and return its notebook_id."""
-        path, name = self.increment_filename('Untitled')
-        notebook_id = self.new_notebook_id(name)
-        metadata = current.new_metadata(name=name)
-        nb = current.new_notebook(metadata=metadata)
-        with open(path,'w') as f:
-            current.write(nb, f, u'json')
-        return notebook_id
-
-    def copy_notebook(self, notebook_id):
-        """Copy an existing notebook and return its notebook_id."""
-        last_mod, nb = self.get_notebook_object(notebook_id)
-        name = nb.metadata.name + '-Copy'
-        path, name = self.increment_filename(name)
-        nb.metadata.name = name
-        notebook_id = self.new_notebook_id(name)
-        self.save_notebook_object(notebook_id, nb)
-        return notebook_id
+        return name

IPython/frontend/html/notebook/notebookapp.py

@@ -1,606 +1,615 b''
 # coding: utf-8
 """A tornado based IPython notebook server.
 
 Authors:
 
 * Brian Granger
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import errno
 import logging
 import os
 import random
 import re
 import select
 import signal
 import socket
 import sys
 import threading
 import time
 import webbrowser
 
 # Third party
 import zmq
 
 # Install the pyzmq ioloop. This has to be done before anything else from
 # tornado is imported.
 from zmq.eventloop import ioloop
 ioloop.install()
 
 from tornado import httpserver
 from tornado import web
 
 # Our own libraries
 from .kernelmanager import MappingKernelManager
 from .handlers import (LoginHandler, LogoutHandler,
     ProjectDashboardHandler, NewHandler, NamedNotebookHandler,
     MainKernelHandler, KernelHandler, KernelActionHandler, IOPubHandler,
     ShellHandler, NotebookRootHandler, NotebookHandler, NotebookCopyHandler,
     RSTHandler, AuthenticatedFileHandler, PrintNotebookHandler,
     MainClusterHandler, ClusterProfileHandler, ClusterActionHandler,
     FileFindHandler,
 )
 from .notebookmanager import NotebookManager
 from .clustermanager import ClusterManager
 
 from IPython.config.application import catch_config_error, boolean_flag
 from IPython.core.application import BaseIPythonApplication
 from IPython.core.profiledir import ProfileDir
 from IPython.frontend.consoleapp import IPythonConsoleApp
 from IPython.lib.kernel import swallow_argv
 from IPython.zmq.session import Session, default_secure
 from IPython.zmq.zmqshell import ZMQInteractiveShell
 from IPython.zmq.ipkernel import (
     flags as ipkernel_flags,
     aliases as ipkernel_aliases,
     IPKernelApp
 )
-from IPython.utils.traitlets import Dict, Unicode, Integer, List, Enum, Bool
+from IPython.utils.importstring import import_item
+from IPython.utils.traitlets import (
+    Dict, Unicode, Integer, List, Enum, Bool,
+    DottedObjectName
+)
 from IPython.utils import py3compat
 from IPython.utils.path import filefind
 
 #-----------------------------------------------------------------------------
 # Module globals
 #-----------------------------------------------------------------------------
 
 _kernel_id_regex = r"(?P<kernel_id>\w+-\w+-\w+-\w+-\w+)"
 _kernel_action_regex = r"(?P<action>restart|interrupt)"
 _notebook_id_regex = r"(?P<notebook_id>\w+-\w+-\w+-\w+-\w+)"
 _profile_regex = r"(?P<profile>[^\/]+)" # there is almost no text that is invalid
 _cluster_action_regex = r"(?P<action>start|stop)"
 
 
 LOCALHOST = '127.0.0.1'
 
 _examples = """
 ipython notebook                       # start the notebook
 ipython notebook --profile=sympy       # use the sympy profile
 ipython notebook --pylab=inline        # pylab in inline plotting mode
 ipython notebook --certfile=mycert.pem # use SSL/TLS certificate
 ipython notebook --port=5555 --ip=*    # Listen on port 5555, all interfaces
 """
 
 #-----------------------------------------------------------------------------
 # Helper functions
 #-----------------------------------------------------------------------------
 
 def url_path_join(a,b):
     if a.endswith('/') and b.startswith('/'):
         return a[:-1]+b
     else:
         return a+b
 
 def random_ports(port, n):
     """Generate a list of n random ports near the given port.
 
     The first 5 ports will be sequential, and the remaining n-5 will be
     randomly selected in the range [port-2*n, port+2*n].
     """
     for i in range(min(5, n)):
         yield port + i
     for i in range(n-5):
         yield port + random.randint(-2*n, 2*n)
 
 #-----------------------------------------------------------------------------
 # The Tornado web application
 #-----------------------------------------------------------------------------
 
 class NotebookWebApplication(web.Application):
 
     def __init__(self, ipython_app, kernel_manager, notebook_manager, 
                  cluster_manager, log,
                  base_project_url, settings_overrides):
         handlers = [
             (r"/", ProjectDashboardHandler),
             (r"/login", LoginHandler),
             (r"/logout", LogoutHandler),
             (r"/new", NewHandler),
             (r"/%s" % _notebook_id_regex, NamedNotebookHandler),
             (r"/%s/copy" % _notebook_id_regex, NotebookCopyHandler),
             (r"/%s/print" % _notebook_id_regex, PrintNotebookHandler),
             (r"/kernels", MainKernelHandler),
             (r"/kernels/%s" % _kernel_id_regex, KernelHandler),
             (r"/kernels/%s/%s" % (_kernel_id_regex, _kernel_action_regex), KernelActionHandler),
             (r"/kernels/%s/iopub" % _kernel_id_regex, IOPubHandler),
             (r"/kernels/%s/shell" % _kernel_id_regex, ShellHandler),
             (r"/notebooks", NotebookRootHandler),
             (r"/notebooks/%s" % _notebook_id_regex, NotebookHandler),
             (r"/rstservice/render", RSTHandler),
             (r"/files/(.*)", AuthenticatedFileHandler, {'path' : notebook_manager.notebook_dir}),
             (r"/clusters", MainClusterHandler),
             (r"/clusters/%s/%s" % (_profile_regex, _cluster_action_regex), ClusterActionHandler),
             (r"/clusters/%s" % _profile_regex, ClusterProfileHandler),
         ]
 
         # Python < 2.6.5 doesn't accept unicode keys in f(**kwargs), and
         # base_project_url will always be unicode, which will in turn
         # make the patterns unicode, and ultimately result in unicode
         # keys in kwargs to handler._execute(**kwargs) in tornado.
         # This enforces that base_project_url be ascii in that situation.
         # 
         # Note that the URLs these patterns check against are escaped,
         # and thus guaranteed to be ASCII: 'héllo' is really 'h%C3%A9llo'.
         base_project_url = py3compat.unicode_to_str(base_project_url, 'ascii')
         
         settings = dict(
             template_path=os.path.join(os.path.dirname(__file__), "templates"),
             static_path=ipython_app.static_file_path,
             static_handler_class = FileFindHandler,
             cookie_secret=os.urandom(1024),
             login_url="%s/login"%(base_project_url.rstrip('/')),
         )
 
         # allow custom overrides for the tornado web app.
         settings.update(settings_overrides)
 
         # prepend base_project_url onto the patterns that we match
         new_handlers = []
         for handler in handlers:
             pattern = url_path_join(base_project_url, handler[0])
             new_handler = tuple([pattern]+list(handler[1:]))
             new_handlers.append( new_handler )
 
         super(NotebookWebApplication, self).__init__(new_handlers, **settings)
 
         self.kernel_manager = kernel_manager
         self.notebook_manager = notebook_manager
         self.cluster_manager = cluster_manager
         self.ipython_app = ipython_app
         self.read_only = self.ipython_app.read_only
         self.log = log
 
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags
 #-----------------------------------------------------------------------------
 
 flags = dict(ipkernel_flags)
 flags['no-browser']=(
     {'NotebookApp' : {'open_browser' : False}},
     "Don't open the notebook in a browser after startup."
 )
 flags['no-mathjax']=(
     {'NotebookApp' : {'enable_mathjax' : False}},
     """Disable MathJax
     
     MathJax is the javascript library IPython uses to render math/LaTeX. It is
     very large, so you may want to disable it if you have a slow internet
     connection, or for offline use of the notebook.
     
     When disabled, equations etc. will appear as their untransformed TeX source.
     """
 )
 flags['read-only'] = (
     {'NotebookApp' : {'read_only' : True}},
     """Allow read-only access to notebooks.
     
     When using a password to protect the notebook server, this flag
     allows unauthenticated clients to view the notebook list, and
     individual notebooks, but not edit them, start kernels, or run
     code.
     
     If no password is set, the server will be entirely read-only.
     """
 )
 
 # Add notebook manager flags
 flags.update(boolean_flag('script', 'NotebookManager.save_script',
                'Auto-save a .py script everytime the .ipynb notebook is saved',
                'Do not auto-save .py scripts for every notebook'))
 
 # the flags that are specific to the frontend
 # these must be scrubbed before being passed to the kernel,
 # or it will raise an error on unrecognized flags
 notebook_flags = ['no-browser', 'no-mathjax', 'read-only', 'script', 'no-script']
 
 aliases = dict(ipkernel_aliases)
 
 aliases.update({
     'ip': 'NotebookApp.ip',
     'port': 'NotebookApp.port',
     'port-retries': 'NotebookApp.port_retries',
     'keyfile': 'NotebookApp.keyfile',
     'certfile': 'NotebookApp.certfile',
     'notebook-dir': 'NotebookManager.notebook_dir',
     'browser': 'NotebookApp.browser',
 })
 
 # remove ipkernel flags that are singletons, and don't make sense in
 # multi-kernel evironment:
 aliases.pop('f', None)
 
 notebook_aliases = [u'port', u'port-retries', u'ip', u'keyfile', u'certfile',
                     u'notebook-dir']
 
 #-----------------------------------------------------------------------------
 # NotebookApp
 #-----------------------------------------------------------------------------
 
 class NotebookApp(BaseIPythonApplication):
 
     name = 'ipython-notebook'
     default_config_file_name='ipython_notebook_config.py'
     
     description = """
         The IPython HTML Notebook.
         
         This launches a Tornado based HTML Notebook Server that serves up an
         HTML5/Javascript Notebook client.
     """
     examples = _examples
     
     classes = IPythonConsoleApp.classes + [MappingKernelManager, NotebookManager]
     flags = Dict(flags)
     aliases = Dict(aliases)
 
     kernel_argv = List(Unicode)
 
     log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'),
                     default_value=logging.INFO,
                     config=True,
                     help="Set the log level by value or name.")
 
     # create requested profiles by default, if they don't exist:
     auto_create = Bool(True)
 
     # file to be opened in the notebook server
     file_to_run = Unicode('')
 
     # Network related information.
 
     ip = Unicode(LOCALHOST, config=True,
         help="The IP address the notebook server will listen on."
     )
 
     def _ip_changed(self, name, old, new):
         if new == u'*': self.ip = u''
 
     port = Integer(8888, config=True,
         help="The port the notebook server will listen on."
     )
     port_retries = Integer(50, config=True,
         help="The number of additional ports to try if the specified port is not available."
     )
 
     certfile = Unicode(u'', config=True, 
         help="""The full path to an SSL/TLS certificate file."""
     )
     
     keyfile = Unicode(u'', config=True, 
         help="""The full path to a private key file for usage with SSL/TLS."""
     )
 
     password = Unicode(u'', config=True,
                       help="""Hashed password to use for web authentication.
 
                       To generate, type in a python/IPython shell:
 
                         from IPython.lib import passwd; passwd()
 
                       The string should be of the form type:salt:hashed-password.
                       """
     )
 
     open_browser = Bool(True, config=True,
                         help="""Whether to open in a browser after starting.
                         The specific browser used is platform dependent and
                         determined by the python standard library `webbrowser`
                         module, unless it is overridden using the --browser
                         (NotebookApp.browser) configuration option.
                         """)
 
     browser = Unicode(u'', config=True,
                       help="""Specify what command to use to invoke a web
                       browser when opening the notebook. If not specified, the
                       default browser will be determined by the `webbrowser`
                       standard library module, which allows setting of the
                       BROWSER environment variable to override it.
                       """)
     
     read_only = Bool(False, config=True,
         help="Whether to prevent editing/execution of notebooks."
     )
     
     webapp_settings = Dict(config=True,
             help="Supply overrides for the tornado.web.Application that the "
                  "IPython notebook uses.")
     
     enable_mathjax = Bool(True, config=True,
         help="""Whether to enable MathJax for typesetting math/TeX
 
         MathJax is the javascript library IPython uses to render math/LaTeX. It is
         very large, so you may want to disable it if you have a slow internet
         connection, or for offline use of the notebook.
 
         When disabled, equations etc. will appear as their untransformed TeX source.
         """
     )
     def _enable_mathjax_changed(self, name, old, new):
         """set mathjax url to empty if mathjax is disabled"""
         if not new:
             self.mathjax_url = u''
 
     base_project_url = Unicode('/', config=True,
                                help='''The base URL for the notebook server''')
     base_kernel_url = Unicode('/', config=True,
                                help='''The base URL for the kernel server''')
     websocket_host = Unicode("", config=True,
         help="""The hostname for the websocket server."""
     )
     
     extra_static_paths = List(Unicode, config=True,
         help="""Extra paths to search for serving static files.
         
         This allows adding javascript/css to be available from the notebook server machine,
         or overriding individual files in the IPython"""
     )
     def _extra_static_paths_default(self):
         return [os.path.join(self.profile_dir.location, 'static')]
     
     @property
     def static_file_path(self):
         """return extra paths + the default location"""
         return self.extra_static_paths + [os.path.join(os.path.dirname(__file__), "static")]
 
     mathjax_url = Unicode("", config=True,
         help="""The url for MathJax.js."""
     )
     def _mathjax_url_default(self):
         if not self.enable_mathjax:
             return u''
         static_url_prefix = self.webapp_settings.get("static_url_prefix",
                                                      "/static/")
         try:
             mathjax = filefind(os.path.join('mathjax', 'MathJax.js'), self.static_file_path)
         except IOError:
             if self.certfile:
                 # HTTPS: load from Rackspace CDN, because SSL certificate requires it
                 base = u"https://c328740.ssl.cf1.rackcdn.com"
             else:
                 base = u"http://cdn.mathjax.org"
             
             url = base + u"/mathjax/latest/MathJax.js"
             self.log.info("Using MathJax from CDN: %s", url)
             return url
         else:
             self.log.info("Using local MathJax from %s" % mathjax)
             return static_url_prefix+u"mathjax/MathJax.js"
     
     def _mathjax_url_changed(self, name, old, new):
         if new and not self.enable_mathjax:
             # enable_mathjax=False overrides mathjax_url
             self.mathjax_url = u''
         else:
             self.log.info("Using MathJax: %s", new)
 
+    notebook_manager_class = DottedObjectName('IPython.frontend.html.notebook.notebookmanager.NotebookManager',
+        config=True,
+        help='The notebook manager class to use.')
+
     def parse_command_line(self, argv=None):
         super(NotebookApp, self).parse_command_line(argv)
         if argv is None:
             argv = sys.argv[1:]
 
         # Scrub frontend-specific flags
         self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags)
         # Kernel should inherit default config file from frontend
         self.kernel_argv.append("--KernelApp.parent_appname='%s'"%self.name)
 
         if self.extra_args:
             f = os.path.abspath(self.extra_args[0])
             if os.path.isdir(f):
                 nbdir = f
             else:
                 self.file_to_run = f
                 nbdir = os.path.dirname(f)
             self.config.NotebookManager.notebook_dir = nbdir
 
     def init_configurables(self):
         # force Session default to be secure
         default_secure(self.config)
         self.kernel_manager = MappingKernelManager(
             config=self.config, log=self.log, kernel_argv=self.kernel_argv,
             connection_dir = self.profile_dir.security_dir,
         )
-        self.notebook_manager = NotebookManager(config=self.config, log=self.log)
+        kls = import_item(self.notebook_manager_class)
+        self.notebook_manager = kls(config=self.config, log=self.log)
         self.log.info("Serving notebooks from %s", self.notebook_manager.notebook_dir)
-        self.notebook_manager.list_notebooks()
+        self.notebook_manager.load_notebook_names()
         self.cluster_manager = ClusterManager(config=self.config, log=self.log)
         self.cluster_manager.update_profiles()
 
     def init_logging(self):
         # This prevents double log messages because tornado use a root logger that
         # self.log is a child of. The logging module dipatches log messages to a log
         # and all of its ancenstors until propagate is set to False.
         self.log.propagate = False
     
     def init_webapp(self):
         """initialize tornado webapp and httpserver"""
         self.web_app = NotebookWebApplication(
             self, self.kernel_manager, self.notebook_manager, 
             self.cluster_manager, self.log,
             self.base_project_url, self.webapp_settings
         )
         if self.certfile:
             ssl_options = dict(certfile=self.certfile)
             if self.keyfile:
                 ssl_options['keyfile'] = self.keyfile
         else:
             ssl_options = None
         self.web_app.password = self.password
         self.http_server = httpserver.HTTPServer(self.web_app, ssl_options=ssl_options)
         if ssl_options is None and not self.ip and not (self.read_only and not self.password):
             self.log.critical('WARNING: the notebook server is listening on all IP addresses '
                               'but not using any encryption or authentication. This is highly '
                               'insecure and not recommended.')
 
         success = None
         for port in random_ports(self.port, self.port_retries+1):
             try:
                 self.http_server.listen(port, self.ip)
             except socket.error as e:
                 if e.errno != errno.EADDRINUSE:
                     raise
                 self.log.info('The port %i is already in use, trying another random port.' % port)
             else:
                 self.port = port
                 success = True
                 break
         if not success:
             self.log.critical('ERROR: the notebook server could not be started because '
                               'no available port could be found.')
             self.exit(1)
     
     def init_signal(self):
         # FIXME: remove this check when pyzmq dependency is >= 2.1.11
         # safely extract zmq version info:
         try:
             zmq_v = zmq.pyzmq_version_info()
         except AttributeError:
             zmq_v = [ int(n) for n in re.findall(r'\d+', zmq.__version__) ]
             if 'dev' in zmq.__version__:
                 zmq_v.append(999)
             zmq_v = tuple(zmq_v)
         if zmq_v >= (2,1,9) and not sys.platform.startswith('win'):
             # This won't work with 2.1.7 and
             # 2.1.9-10 will log ugly 'Interrupted system call' messages,
             # but it will work
             signal.signal(signal.SIGINT, self._handle_sigint)
         signal.signal(signal.SIGTERM, self._signal_stop)
     
     def _handle_sigint(self, sig, frame):
         """SIGINT handler spawns confirmation dialog"""
         # register more forceful signal handler for ^C^C case
         signal.signal(signal.SIGINT, self._signal_stop)
         # request confirmation dialog in bg thread, to avoid
         # blocking the App
         thread = threading.Thread(target=self._confirm_exit)
         thread.daemon = True
         thread.start()
     
     def _restore_sigint_handler(self):
         """callback for restoring original SIGINT handler"""
         signal.signal(signal.SIGINT, self._handle_sigint)
     
     def _confirm_exit(self):
         """confirm shutdown on ^C
         
         A second ^C, or answering 'y' within 5s will cause shutdown,
         otherwise original SIGINT handler will be restored.
         
         This doesn't work on Windows.
         """
         # FIXME: remove this delay when pyzmq dependency is >= 2.1.11
         time.sleep(0.1)
         sys.stdout.write("Shutdown Notebook Server (y/[n])? ")
         sys.stdout.flush()
         r,w,x = select.select([sys.stdin], [], [], 5)
         if r:
             line = sys.stdin.readline()
             if line.lower().startswith('y'):
                 self.log.critical("Shutdown confirmed")
                 ioloop.IOLoop.instance().stop()
                 return
         else:
             print "No answer for 5s:",
         print "resuming operation..."
         # no answer, or answer is no:
         # set it back to original SIGINT handler
         # use IOLoop.add_callback because signal.signal must be called
         # from main thread
         ioloop.IOLoop.instance().add_callback(self._restore_sigint_handler)
     
     def _signal_stop(self, sig, frame):
         self.log.critical("received signal %s, stopping", sig)
         ioloop.IOLoop.instance().stop()
     
     @catch_config_error
     def initialize(self, argv=None):
         self.init_logging()
         super(NotebookApp, self).initialize(argv)
         self.init_configurables()
         self.init_webapp()
         self.init_signal()
 
     def cleanup_kernels(self):
         """shutdown all kernels
         
         The kernels will shutdown themselves when this process no longer exists,
         but explicit shutdown allows the KernelManagers to cleanup the connection files.
         """
         self.log.info('Shutting down kernels')
         km = self.kernel_manager
         # copy list, since shutdown_kernel deletes keys
         for kid in list(km.kernel_ids):
             km.shutdown_kernel(kid)
 
     def start(self):
         ip = self.ip if self.ip else '[all ip addresses on your system]'
         proto = 'https' if self.certfile else 'http'
         info = self.log.info
         info("The IPython Notebook is running at: %s://%s:%i%s" %
              (proto, ip, self.port,self.base_project_url) )
         info("Use Control-C to stop this server and shut down all kernels.")
 
         if self.open_browser or self.file_to_run:
             ip = self.ip or '127.0.0.1'
             try:
                 browser = webbrowser.get(self.browser or None)
             except webbrowser.Error as e:
                 self.log.warn('No web browser found: %s.' % e)
                 browser = None
 
             if self.file_to_run:
                 name, _ = os.path.splitext(os.path.basename(self.file_to_run))
                 url = self.notebook_manager.rev_mapping.get(name, '')
             else:
                 url = ''
             if browser:
                 b = lambda : browser.open("%s://%s:%i%s%s" % (proto, ip,
                     self.port, self.base_project_url, url), new=2)
                 threading.Thread(target=b).start()
         try:
             ioloop.IOLoop.instance().start()
         except KeyboardInterrupt:
             info("Interrupted...")
         finally:
             self.cleanup_kernels()
     
 
 #-----------------------------------------------------------------------------
 # Main entry point
 #-----------------------------------------------------------------------------
 
 def launch_new_instance():
     app = NotebookApp.instance()
     app.initialize()
     app.start()
 

IPython/frontend/html/notebook/tests/test_nbmanager.py

@@ -1,34 +1,34 b''
 """Tests for the notebook manager."""
 
 import os
 from unittest import TestCase
 from tempfile import NamedTemporaryFile
 
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.traitlets import TraitError
 
-from IPython.frontend.html.notebook.notebookmanager import NotebookManager
+from IPython.frontend.html.notebook.filenbmanager import FileNotebookManager
 
 class TestNotebookManager(TestCase):
 
     def test_nb_dir(self):
         with TemporaryDirectory() as td:
-            km = NotebookManager(notebook_dir=td)
-            self.assertEqual(km.notebook_dir, td)
+            km = FileNotebookManager(notebook_dir=td)
+            self.assertEquals(km.notebook_dir, td)
 
     def test_create_nb_dir(self):
         with TemporaryDirectory() as td:
             nbdir = os.path.join(td, 'notebooks')
-            km = NotebookManager(notebook_dir=nbdir)
-            self.assertEqual(km.notebook_dir, nbdir)
+            km = FileNotebookManager(notebook_dir=nbdir)
+            self.assertEquals(km.notebook_dir, nbdir)
 
     def test_missing_nb_dir(self):
         with TemporaryDirectory() as td:
             nbdir = os.path.join(td, 'notebook', 'dir', 'is', 'missing')
-            self.assertRaises(TraitError, NotebookManager, notebook_dir=nbdir)
+            self.assertRaises(TraitError, FileNotebookManager, notebook_dir=nbdir)
 
     def test_invalid_nb_dir(self):
         with NamedTemporaryFile() as tf:
-            self.assertRaises(TraitError, NotebookManager, notebook_dir=tf.name)
+            self.assertRaises(TraitError, FileNotebookManager, notebook_dir=tf.name)