Allow specifying format when getting files from contents API

IPython/html/services/contents/filemanager.py

@@ -1,545 +1,559 b''
 """A contents manager that uses the local file system for storage."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import base64
 import io
 import os
 import glob
 import shutil
 
 from tornado import web
 
 from .manager import ContentsManager
 from IPython import nbformat
 from IPython.utils.io import atomic_writing
 from IPython.utils.path import ensure_dir_exists
 from IPython.utils.traitlets import Unicode, Bool, TraitError
 from IPython.utils.py3compat import getcwd
 from IPython.utils import tz
 from IPython.html.utils import is_hidden, to_os_path, url_path_join
 
 
 class FileContentsManager(ContentsManager):
 
     root_dir = Unicode(getcwd(), config=True)
 
     save_script = Bool(False, config=True, help='DEPRECATED, IGNORED')
     def _save_script_changed(self):
         self.log.warn("""
         Automatically saving notebooks as scripts has been removed.
         Use `ipython nbconvert --to python [notebook]` instead.
         """)
 
     def _root_dir_changed(self, name, old, new):
         """Do a bit of validation of the root_dir."""
         if not os.path.isabs(new):
             # If we receive a non-absolute path, make it absolute.
             self.root_dir = os.path.abspath(new)
             return
         if not os.path.isdir(new):
             raise TraitError("%r is not a directory" % new)
 
     checkpoint_dir = Unicode('.ipynb_checkpoints', config=True,
         help="""The directory name in which to keep file checkpoints
 
         This is a path relative to the file's own directory.
 
         By default, it is .ipynb_checkpoints
         """
     )
 
     def _copy(self, src, dest):
         """copy src to dest
 
         like shutil.copy2, but log errors in copystat
         """
         shutil.copyfile(src, dest)
         try:
             shutil.copystat(src, dest)
         except OSError as e:
             self.log.debug("copystat on %s failed", dest, exc_info=True)
 
     def _get_os_path(self, path):
         """Given an API path, return its file system path.
 
         Parameters
         ----------
         path : string
             The relative API path to the named file.
 
         Returns
         -------
         path : string
             Native, absolute OS path to for a file.
         """
         return to_os_path(path, self.root_dir)
 
     def dir_exists(self, path):
         """Does the API-style path refer to an extant directory?
 
         API-style wrapper for os.path.isdir
 
         Parameters
         ----------
         path : string
             The path to check. This is an API path (`/` separated,
             relative to root_dir).
 
         Returns
         -------
         exists : bool
             Whether the path is indeed a directory.
         """
         path = path.strip('/')
         os_path = self._get_os_path(path=path)
         return os.path.isdir(os_path)
 
     def is_hidden(self, path):
         """Does the API style path correspond to a hidden directory or file?
 
         Parameters
         ----------
         path : string
             The path to check. This is an API path (`/` separated,
             relative to root_dir).
 
         Returns
         -------
         hidden : bool
             Whether the path exists and is hidden.
         """
         path = path.strip('/')
         os_path = self._get_os_path(path=path)
         return is_hidden(os_path, self.root_dir)
 
     def file_exists(self, path):
         """Returns True if the file exists, else returns False.
 
         API-style wrapper for os.path.isfile
 
         Parameters
         ----------
         path : string
             The relative path to the file (with '/' as separator)
 
         Returns
         -------
         exists : bool
             Whether the file exists.
         """
         path = path.strip('/')
         os_path = self._get_os_path(path)
         return os.path.isfile(os_path)
 
     def exists(self, path):
         """Returns True if the path exists, else returns False.
 
         API-style wrapper for os.path.exists
 
         Parameters
         ----------
         path : string
             The API path to the file (with '/' as separator)
 
         Returns
         -------
         exists : bool
             Whether the target exists.
         """
         path = path.strip('/')
         os_path = self._get_os_path(path=path)
         return os.path.exists(os_path)
 
     def _base_model(self, path):
         """Build the common base of a contents model"""
         os_path = self._get_os_path(path)
         info = os.stat(os_path)
         last_modified = tz.utcfromtimestamp(info.st_mtime)
         created = tz.utcfromtimestamp(info.st_ctime)
         # Create the base model.
         model = {}
         model['name'] = path.rsplit('/', 1)[-1]
         model['path'] = path
         model['last_modified'] = last_modified
         model['created'] = created
         model['content'] = None
         model['format'] = None
         return model
 
     def _dir_model(self, path, content=True):
         """Build a model for a directory
 
         if content is requested, will include a listing of the directory
         """
         os_path = self._get_os_path(path)
 
         four_o_four = u'directory does not exist: %r' % os_path
 
         if not os.path.isdir(os_path):
             raise web.HTTPError(404, four_o_four)
         elif is_hidden(os_path, self.root_dir):
             self.log.info("Refusing to serve hidden directory %r, via 404 Error",
                 os_path
             )
             raise web.HTTPError(404, four_o_four)
 
         model = self._base_model(path)
         model['type'] = 'directory'
         if content:
             model['content'] = contents = []
             os_dir = self._get_os_path(path)
             for name in os.listdir(os_dir):
                 os_path = os.path.join(os_dir, name)
                 # skip over broken symlinks in listing
                 if not os.path.exists(os_path):
                     self.log.warn("%s doesn't exist", os_path)
                     continue
                 elif not os.path.isfile(os_path) and not os.path.isdir(os_path):
                     self.log.debug("%s not a regular file", os_path)
                     continue
                 if self.should_list(name) and not is_hidden(os_path, self.root_dir):
                     contents.append(self.get_model(
                         path='%s/%s' % (path, name),
                         content=False)
                     )
 
             model['format'] = 'json'
 
         return model
 
-    def _file_model(self, path, content=True):
+    def _file_model(self, path, content=True, format=None):
         """Build a model for a file
 
         if content is requested, include the file contents.
-        UTF-8 text files will be unicode, binary files will be base64-encoded.
+
+        format:
+          If 'text', the contents will be decoded as UTF-8.
+          If 'base64', the raw bytes contents will be encoded as base64.
+          If not specified, try to decode as UTF-8, and fall back to base64
         """
         model = self._base_model(path)
         model['type'] = 'file'
         if content:
             os_path = self._get_os_path(path)
             if not os.path.isfile(os_path):
                 # could be FIFO
                 raise web.HTTPError(400, "Cannot get content of non-file %s" % os_path)
             with io.open(os_path, 'rb') as f:
                 bcontent = f.read()
+
+            if format != 'base64':
             try:
                 model['content'] = bcontent.decode('utf8')
             except UnicodeError as e:
-                model['content'] = base64.encodestring(bcontent).decode('ascii')
-                model['format'] = 'base64'
+                    if format == 'text':
+                        raise web.HTTPError(400, "%s is not UTF-8 encoded" % path)
             else:
                 model['format'] = 'text'
+
+            if model['content'] is None:
+                model['content'] = base64.encodestring(bcontent).decode('ascii')
+                model['format'] = 'base64'
+
         return model
 
 
     def _notebook_model(self, path, content=True):
         """Build a notebook model
 
         if content is requested, the notebook content will be populated
         as a JSON structure (not double-serialized)
         """
         model = self._base_model(path)
         model['type'] = 'notebook'
         if content:
             os_path = self._get_os_path(path)
             with io.open(os_path, 'r', encoding='utf-8') as f:
                 try:
                     nb = nbformat.read(f, as_version=4)
                 except Exception as e:
                     raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e))
             self.mark_trusted_cells(nb, path)
             model['content'] = nb
             model['format'] = 'json'
             self.validate_notebook_model(model)
         return model
 
-    def get_model(self, path, content=True, type_=None):
+    def get_model(self, path, content=True, type_=None, format=None):
         """ Takes a path for an entity and returns its model
 
         Parameters
         ----------
         path : str
             the API path that describes the relative path for the target
         content : bool
             Whether to include the contents in the reply
         type_ : str, optional
             The requested type - 'file', 'notebook', or 'directory'.
             Will raise HTTPError 406 if the content doesn't match.
+        format : str, optional
+            The requested format for file contents. 'text' or 'base64'.
+            Ignored if this returns a notebook or directory model.
 
         Returns
         -------
         model : dict
             the contents model. If content=True, returns the contents
             of the file or directory as well.
         """
         path = path.strip('/')
 
         if not self.exists(path):
             raise web.HTTPError(404, u'No such file or directory: %s' % path)
 
         os_path = self._get_os_path(path)
         if os.path.isdir(os_path):
             if type_ not in (None, 'directory'):
                 raise web.HTTPError(400,
                                 u'%s is a directory, not a %s' % (path, type_))
             model = self._dir_model(path, content=content)
         elif type_ == 'notebook' or (type_ is None and path.endswith('.ipynb')):
             model = self._notebook_model(path, content=content)
         else:
             if type_ == 'directory':
                 raise web.HTTPError(400,
                                 u'%s is not a directory')
-            model = self._file_model(path, content=content)
+            model = self._file_model(path, content=content, format=format)
         return model
 
     def _save_notebook(self, os_path, model, path=''):
         """save a notebook file"""
         # Save the notebook file
         nb = nbformat.from_dict(model['content'])
 
         self.check_and_sign(nb, path)
 
         with atomic_writing(os_path, encoding='utf-8') as f:
             nbformat.write(nb, f, version=nbformat.NO_CONVERT)
 
     def _save_file(self, os_path, model, path=''):
         """save a non-notebook file"""
         fmt = model.get('format', None)
         if fmt not in {'text', 'base64'}:
             raise web.HTTPError(400, "Must specify format of file contents as 'text' or 'base64'")
         try:
             content = model['content']
             if fmt == 'text':
                 bcontent = content.encode('utf8')
             else:
                 b64_bytes = content.encode('ascii')
                 bcontent = base64.decodestring(b64_bytes)
         except Exception as e:
             raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e))
         with atomic_writing(os_path, text=False) as f:
             f.write(bcontent)
 
     def _save_directory(self, os_path, model, path=''):
         """create a directory"""
         if is_hidden(os_path, self.root_dir):
             raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path)
         if not os.path.exists(os_path):
             os.mkdir(os_path)
         elif not os.path.isdir(os_path):
             raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
         else:
             self.log.debug("Directory %r already exists", os_path)
 
     def save(self, model, path=''):
         """Save the file model and return the model with no content."""
         path = path.strip('/')
 
         if 'type' not in model:
             raise web.HTTPError(400, u'No file type provided')
         if 'content' not in model and model['type'] != 'directory':
             raise web.HTTPError(400, u'No file content provided')
 
         # One checkpoint should always exist
         if self.file_exists(path) and not self.list_checkpoints(path):
             self.create_checkpoint(path)
 
         os_path = self._get_os_path(path)
         self.log.debug("Saving %s", os_path)
         try:
             if model['type'] == 'notebook':
                 self._save_notebook(os_path, model, path)
             elif model['type'] == 'file':
                 self._save_file(os_path, model, path)
             elif model['type'] == 'directory':
                 self._save_directory(os_path, model, path)
             else:
                 raise web.HTTPError(400, "Unhandled contents type: %s" % model['type'])
         except web.HTTPError:
             raise
         except Exception as e:
             raise web.HTTPError(400, u'Unexpected error while saving file: %s %s' % (os_path, e))
 
         validation_message = None
         if model['type'] == 'notebook':
             self.validate_notebook_model(model)
             validation_message = model.get('message', None)
 
         model = self.get_model(path, content=False)
         if validation_message:
             model['message'] = validation_message
         return model
 
     def update(self, model, path):
         """Update the file's path
 
         For use in PATCH requests, to enable renaming a file without
         re-uploading its contents. Only used for renaming at the moment.
         """
         path = path.strip('/')
         new_path = model.get('path', path).strip('/')
         if path != new_path:
             self.rename(path, new_path)
         model = self.get_model(new_path, content=False)
         return model
 
     def delete(self, path):
         """Delete file at path."""
         path = path.strip('/')
         os_path = self._get_os_path(path)
         rm = os.unlink
         if os.path.isdir(os_path):
             listing = os.listdir(os_path)
             # don't delete non-empty directories (checkpoints dir doesn't count)
             if listing and listing != [self.checkpoint_dir]:
                 raise web.HTTPError(400, u'Directory %s not empty' % os_path)
         elif not os.path.isfile(os_path):
             raise web.HTTPError(404, u'File does not exist: %s' % os_path)
 
         # clear checkpoints
         for checkpoint in self.list_checkpoints(path):
             checkpoint_id = checkpoint['id']
             cp_path = self.get_checkpoint_path(checkpoint_id, path)
             if os.path.isfile(cp_path):
                 self.log.debug("Unlinking checkpoint %s", cp_path)
                 os.unlink(cp_path)
 
         if os.path.isdir(os_path):
             self.log.debug("Removing directory %s", os_path)
             shutil.rmtree(os_path)
         else:
             self.log.debug("Unlinking file %s", os_path)
             rm(os_path)
 
     def rename(self, old_path, new_path):
         """Rename a file."""
         old_path = old_path.strip('/')
         new_path = new_path.strip('/')
         if new_path == old_path:
             return
 
         new_os_path = self._get_os_path(new_path)
         old_os_path = self._get_os_path(old_path)
 
         # Should we proceed with the move?
         if os.path.exists(new_os_path):
             raise web.HTTPError(409, u'File already exists: %s' % new_path)
 
         # Move the file
         try:
             shutil.move(old_os_path, new_os_path)
         except Exception as e:
             raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e))
 
         # Move the checkpoints
         old_checkpoints = self.list_checkpoints(old_path)
         for cp in old_checkpoints:
             checkpoint_id = cp['id']
             old_cp_path = self.get_checkpoint_path(checkpoint_id, old_path)
             new_cp_path = self.get_checkpoint_path(checkpoint_id, new_path)
             if os.path.isfile(old_cp_path):
                 self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path)
                 shutil.move(old_cp_path, new_cp_path)
 
     # Checkpoint-related utilities
 
     def get_checkpoint_path(self, checkpoint_id, path):
         """find the path to a checkpoint"""
         path = path.strip('/')
         parent, name = ('/' + path).rsplit('/', 1)
         parent = parent.strip('/')
         basename, ext = os.path.splitext(name)
         filename = u"{name}-{checkpoint_id}{ext}".format(
             name=basename,
             checkpoint_id=checkpoint_id,
             ext=ext,
         )
         os_path = self._get_os_path(path=parent)
         cp_dir = os.path.join(os_path, self.checkpoint_dir)
         ensure_dir_exists(cp_dir)
         cp_path = os.path.join(cp_dir, filename)
         return cp_path
 
     def get_checkpoint_model(self, checkpoint_id, path):
         """construct the info dict for a given checkpoint"""
         path = path.strip('/')
         cp_path = self.get_checkpoint_path(checkpoint_id, path)
         stats = os.stat(cp_path)
         last_modified = tz.utcfromtimestamp(stats.st_mtime)
         info = dict(
             id = checkpoint_id,
             last_modified = last_modified,
         )
         return info
 
     # public checkpoint API
 
     def create_checkpoint(self, path):
         """Create a checkpoint from the current state of a file"""
         path = path.strip('/')
         if not self.file_exists(path):
             raise web.HTTPError(404)
         src_path = self._get_os_path(path)
         # only the one checkpoint ID:
         checkpoint_id = u"checkpoint"
         cp_path = self.get_checkpoint_path(checkpoint_id, path)
         self.log.debug("creating checkpoint for %s", path)
         self._copy(src_path, cp_path)
 
         # return the checkpoint info
         return self.get_checkpoint_model(checkpoint_id, path)
 
     def list_checkpoints(self, path):
         """list the checkpoints for a given file
 
         This contents manager currently only supports one checkpoint per file.
         """
         path = path.strip('/')
         checkpoint_id = "checkpoint"
         os_path = self.get_checkpoint_path(checkpoint_id, path)
         if not os.path.exists(os_path):
             return []
         else:
             return [self.get_checkpoint_model(checkpoint_id, path)]
 
 
     def restore_checkpoint(self, checkpoint_id, path):
         """restore a file to a checkpointed state"""
         path = path.strip('/')
         self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
         nb_path = self._get_os_path(path)
         cp_path = self.get_checkpoint_path(checkpoint_id, path)
         if not os.path.isfile(cp_path):
             self.log.debug("checkpoint file does not exist: %s", cp_path)
             raise web.HTTPError(404,
                 u'checkpoint does not exist: %s@%s' % (path, checkpoint_id)
             )
         # ensure notebook is readable (never restore from an unreadable notebook)
         if cp_path.endswith('.ipynb'):
             with io.open(cp_path, 'r', encoding='utf-8') as f:
                 nbformat.read(f, as_version=4)
         self._copy(cp_path, nb_path)
         self.log.debug("copying %s -> %s", cp_path, nb_path)
 
     def delete_checkpoint(self, checkpoint_id, path):
         """delete a file's checkpoint"""
         path = path.strip('/')
         cp_path = self.get_checkpoint_path(checkpoint_id, path)
         if not os.path.isfile(cp_path):
             raise web.HTTPError(404,
                 u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id)
             )
         self.log.debug("unlinking %s", cp_path)
         os.unlink(cp_path)
 
     def info_string(self):
         return "Serving notebooks from local directory: %s" % self.root_dir
 
     def get_kernel_path(self, path, model=None):
         """Return the initial working dir a kernel associated with a given notebook"""
         if '/' in path:
             parent_dir = path.rsplit('/', 1)[0]
         else:
             parent_dir = ''
         return self._get_os_path(parent_dir)

IPython/html/services/contents/handlers.py

@@ -1,261 +1,265 b''
 """Tornado handlers for the contents web service."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import json
 
 from tornado import web
 
 from IPython.html.utils import url_path_join, url_escape
 from IPython.utils.jsonutil import date_default
 
 from IPython.html.base.handlers import (
     IPythonHandler, json_errors, path_regex,
 )
 
 
 def sort_key(model):
     """key function for case-insensitive sort by name and type"""
     iname = model['name'].lower()
     type_key = {
         'directory' : '0',
         'notebook'  : '1',
         'file'      : '2',
     }.get(model['type'], '9')
     return u'%s%s' % (type_key, iname)
 
 class ContentsHandler(IPythonHandler):
 
     SUPPORTED_METHODS = (u'GET', u'PUT', u'PATCH', u'POST', u'DELETE')
 
     def location_url(self, path):
         """Return the full URL location of a file.
 
         Parameters
         ----------
         path : unicode
             The API path of the file, such as "foo/bar.txt".
         """
         return url_escape(url_path_join(
             self.base_url, 'api', 'contents', path
         ))
 
     def _finish_model(self, model, location=True):
         """Finish a JSON request with a model, setting relevant headers, etc."""
         if location:
             location = self.location_url(model['path'])
             self.set_header('Location', location)
         self.set_header('Last-Modified', model['last_modified'])
         self.finish(json.dumps(model, default=date_default))
 
     @web.authenticated
     @json_errors
     def get(self, path=''):
         """Return a model for a file or directory.
 
         A directory model contains a list of models (without content)
         of the files and directories it contains.
         """
         path = path or ''
         type_ = self.get_query_argument('type', default=None)
         if type_ not in {None, 'directory', 'file', 'notebook'}:
             raise web.HTTPError(400, u'Type %r is invalid' % type_)
 
-        model = self.contents_manager.get_model(path=path, type_=type_)
+        format = self.get_query_argument('format', default=None)#
+        if format not in {None, 'text', 'base64'}:
+            raise web.HTTPError(400, u'Format %r is invalid' % format)
+
+        model = self.contents_manager.get_model(path=path, type_=type_, format=format)
         if model['type'] == 'directory':
             # group listing by type, then by name (case-insensitive)
             # FIXME: sorting should be done in the frontends
             model['content'].sort(key=sort_key)
         self._finish_model(model, location=False)
 
     @web.authenticated
     @json_errors
     def patch(self, path=''):
         """PATCH renames a file or directory without re-uploading content."""
         cm = self.contents_manager
         model = self.get_json_body()
         if model is None:
             raise web.HTTPError(400, u'JSON body missing')
         model = cm.update(model, path)
         self._finish_model(model)
 
     def _copy(self, copy_from, copy_to=None):
         """Copy a file, optionally specifying a target directory."""
         self.log.info(u"Copying {copy_from} to {copy_to}".format(
             copy_from=copy_from,
             copy_to=copy_to or '',
         ))
         model = self.contents_manager.copy(copy_from, copy_to)
         self.set_status(201)
         self._finish_model(model)
 
     def _upload(self, model, path):
         """Handle upload of a new file to path"""
         self.log.info(u"Uploading file to %s", path)
         model = self.contents_manager.new(model, path)
         self.set_status(201)
         self._finish_model(model)
         
     def _new_untitled(self, path, type='', ext=''):
         """Create a new, empty untitled entity"""
         self.log.info(u"Creating new %s in %s", type or 'file', path)
         model = self.contents_manager.new_untitled(path=path, type=type, ext=ext)
         self.set_status(201)
         self._finish_model(model)
 
     def _save(self, model, path):
         """Save an existing file."""
         self.log.info(u"Saving file at %s", path)
         model = self.contents_manager.save(model, path)
         self._finish_model(model)
 
     @web.authenticated
     @json_errors
     def post(self, path=''):
         """Create a new file in the specified path.
 
         POST creates new files. The server always decides on the name.
 
         POST /api/contents/path
           New untitled, empty file or directory.
         POST /api/contents/path
           with body {"copy_from" : "/path/to/OtherNotebook.ipynb"}
           New copy of OtherNotebook in path
         """
 
         cm = self.contents_manager
 
         if cm.file_exists(path):
             raise web.HTTPError(400, "Cannot POST to files, use PUT instead.")
 
         if not cm.dir_exists(path):
             raise web.HTTPError(404, "No such directory: %s" % path)
 
         model = self.get_json_body()
 
         if model is not None:
             copy_from = model.get('copy_from')
             ext = model.get('ext', '')
             type = model.get('type', '')
             if copy_from:
                 self._copy(copy_from, path)
             else:
                 self._new_untitled(path, type=type, ext=ext)
         else:
             self._new_untitled(path)
 
     @web.authenticated
     @json_errors
     def put(self, path=''):
         """Saves the file in the location specified by name and path.
 
         PUT is very similar to POST, but the requester specifies the name,
         whereas with POST, the server picks the name.
 
         PUT /api/contents/path/Name.ipynb
           Save notebook at ``path/Name.ipynb``. Notebook structure is specified
           in `content` key of JSON request body. If content is not specified,
           create a new empty notebook.
         """
         model = self.get_json_body()
         if model:
             if model.get('copy_from'):
                 raise web.HTTPError(400, "Cannot copy with PUT, only POST")
             if self.contents_manager.file_exists(path):
                 self._save(model, path)
             else:
                 self._upload(model, path)
         else:
             self._new_untitled(path)
 
     @web.authenticated
     @json_errors
     def delete(self, path=''):
         """delete a file in the given path"""
         cm = self.contents_manager
         self.log.warn('delete %s', path)
         cm.delete(path)
         self.set_status(204)
         self.finish()
 
 
 class CheckpointsHandler(IPythonHandler):
 
     SUPPORTED_METHODS = ('GET', 'POST')
 
     @web.authenticated
     @json_errors
     def get(self, path=''):
         """get lists checkpoints for a file"""
         cm = self.contents_manager
         checkpoints = cm.list_checkpoints(path)
         data = json.dumps(checkpoints, default=date_default)
         self.finish(data)
 
     @web.authenticated
     @json_errors
     def post(self, path=''):
         """post creates a new checkpoint"""
         cm = self.contents_manager
         checkpoint = cm.create_checkpoint(path)
         data = json.dumps(checkpoint, default=date_default)
         location = url_path_join(self.base_url, 'api/contents',
             path, 'checkpoints', checkpoint['id'])
         self.set_header('Location', url_escape(location))
         self.set_status(201)
         self.finish(data)
 
 
 class ModifyCheckpointsHandler(IPythonHandler):
 
     SUPPORTED_METHODS = ('POST', 'DELETE')
 
     @web.authenticated
     @json_errors
     def post(self, path, checkpoint_id):
         """post restores a file from a checkpoint"""
         cm = self.contents_manager
         cm.restore_checkpoint(checkpoint_id, path)
         self.set_status(204)
         self.finish()
 
     @web.authenticated
     @json_errors
     def delete(self, path, checkpoint_id):
         """delete clears a checkpoint for a given file"""
         cm = self.contents_manager
         cm.delete_checkpoint(checkpoint_id, path)
         self.set_status(204)
         self.finish()
 
 
 class NotebooksRedirectHandler(IPythonHandler):
     """Redirect /api/notebooks to /api/contents"""
     SUPPORTED_METHODS = ('GET', 'PUT', 'PATCH', 'POST', 'DELETE')
 
     def get(self, path):
         self.log.warn("/api/notebooks is deprecated, use /api/contents")
         self.redirect(url_path_join(
             self.base_url,
             'api/contents',
             path
         ))
 
     put = patch = post = delete = get
 
 
 #-----------------------------------------------------------------------------
 # URL to handler mappings
 #-----------------------------------------------------------------------------
 
 
 _checkpoint_id_regex = r"(?P<checkpoint_id>[\w-]+)"
 
 default_handlers = [
     (r"/api/contents%s/checkpoints" % path_regex, CheckpointsHandler),
     (r"/api/contents%s/checkpoints/%s" % (path_regex, _checkpoint_id_regex),
         ModifyCheckpointsHandler),
     (r"/api/contents%s" % path_regex, ContentsHandler),
     (r"/api/notebooks/?(.*)", NotebooksRedirectHandler),
 ]

IPython/html/services/contents/manager.py

@@ -1,373 +1,373 b''
 """A base class for contents managers."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from fnmatch import fnmatch
 import itertools
 import json
 import os
 
 from tornado.web import HTTPError
 
 from IPython.config.configurable import LoggingConfigurable
 from IPython.nbformat import sign, validate, ValidationError
 from IPython.nbformat.v4 import new_notebook
 from IPython.utils.traitlets import Instance, Unicode, List
 
 
 class ContentsManager(LoggingConfigurable):
     """Base class for serving files and directories.
 
     This serves any text or binary file,
     as well as directories,
     with special handling for JSON notebook documents.
 
     Most APIs take a path argument,
     which is always an API-style unicode path,
     and always refers to a directory.
 
     - unicode, not url-escaped
     - '/'-separated
     - leading and trailing '/' will be stripped
     - if unspecified, path defaults to '',
       indicating the root path.
 
     """
 
     notary = Instance(sign.NotebookNotary)
     def _notary_default(self):
         return sign.NotebookNotary(parent=self)
 
     hide_globs = List(Unicode, [
             u'__pycache__', '*.pyc', '*.pyo',
             '.DS_Store', '*.so', '*.dylib', '*~',
         ], config=True, help="""
         Glob patterns to hide in file and directory listings.
     """)
 
     untitled_notebook = Unicode("Untitled", config=True,
         help="The base name used when creating untitled notebooks."
     )
 
     untitled_file = Unicode("untitled", config=True,
         help="The base name used when creating untitled files."
     )
 
     untitled_directory = Unicode("Untitled Folder", config=True,
         help="The base name used when creating untitled directories."
     )
 
     # ContentsManager API part 1: methods that must be
     # implemented in subclasses.
 
     def dir_exists(self, path):
         """Does the API-style path (directory) actually exist?
 
         Like os.path.isdir
 
         Override this method in subclasses.
 
         Parameters
         ----------
         path : string
             The path to check
 
         Returns
         -------
         exists : bool
             Whether the path does indeed exist.
         """
         raise NotImplementedError
 
     def is_hidden(self, path):
         """Does the API style path correspond to a hidden directory or file?
 
         Parameters
         ----------
         path : string
             The path to check. This is an API path (`/` separated,
             relative to root dir).
 
         Returns
         -------
         hidden : bool
             Whether the path is hidden.
 
         """
         raise NotImplementedError
 
     def file_exists(self, path=''):
         """Does a file exist at the given path?
 
         Like os.path.isfile
 
         Override this method in subclasses.
 
         Parameters
         ----------
         name : string
             The name of the file you are checking.
         path : string
             The relative path to the file's directory (with '/' as separator)
 
         Returns
         -------
         exists : bool
             Whether the file exists.
         """
         raise NotImplementedError('must be implemented in a subclass')
 
     def exists(self, path):
         """Does a file or directory exist at the given path?
 
         Like os.path.exists
 
         Parameters
         ----------
         path : string
             The relative path to the file's directory (with '/' as separator)
 
         Returns
         -------
         exists : bool
             Whether the target exists.
         """
         return self.file_exists(path) or self.dir_exists(path)
 
-    def get_model(self, path, content=True, type_=None):
+    def get_model(self, path, content=True, type_=None, format=None):
         """Get the model of a file or directory with or without content."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def save(self, model, path):
         """Save the file or directory and return the model with no content."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def update(self, model, path):
         """Update the file or directory and return the model with no content.
 
         For use in PATCH requests, to enable renaming a file without
         re-uploading its contents. Only used for renaming at the moment.
         """
         raise NotImplementedError('must be implemented in a subclass')
 
     def delete(self, path):
         """Delete file or directory by path."""
         raise NotImplementedError('must be implemented in a subclass')
 
     def create_checkpoint(self, path):
         """Create a checkpoint of the current state of a file
 
         Returns a checkpoint_id for the new checkpoint.
         """
         raise NotImplementedError("must be implemented in a subclass")
 
     def list_checkpoints(self, path):
         """Return a list of checkpoints for a given file"""
         return []
 
     def restore_checkpoint(self, checkpoint_id, path):
         """Restore a file from one of its checkpoints"""
         raise NotImplementedError("must be implemented in a subclass")
 
     def delete_checkpoint(self, checkpoint_id, path):
         """delete a checkpoint for a file"""
         raise NotImplementedError("must be implemented in a subclass")
 
     # ContentsManager API part 2: methods that have useable default
     # implementations, but can be overridden in subclasses.
 
     def info_string(self):
         return "Serving contents"
 
     def get_kernel_path(self, path, model=None):
         """Return the API path for the kernel
         
         KernelManagers can turn this value into a filesystem path,
         or ignore it altogether.
         """
         return path
 
     def increment_filename(self, filename, path=''):
         """Increment a filename until it is unique.
 
         Parameters
         ----------
         filename : unicode
             The name of a file, including extension
         path : unicode
             The API path of the target's directory
 
         Returns
         -------
         name : unicode
             A filename that is unique, based on the input filename.
         """
         path = path.strip('/')
         basename, ext = os.path.splitext(filename)
         for i in itertools.count():
             name = u'{basename}{i}{ext}'.format(basename=basename, i=i,
                                                 ext=ext)
             if not self.exists(u'{}/{}'.format(path, name)):
                 break
         return name
 
     def validate_notebook_model(self, model):
         """Add failed-validation message to model"""
         try:
             validate(model['content'])
         except ValidationError as e:
             model['message'] = u'Notebook Validation failed: {}:\n{}'.format(
                 e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'),
             )
         return model
     
     def new_untitled(self, path='', type='', ext=''):
         """Create a new untitled file or directory in path
         
         path must be a directory
         
         File extension can be specified.
         
         Use `new` to create files with a fully specified path (including filename).
         """
         path = path.strip('/')
         if not self.dir_exists(path):
             raise HTTPError(404, 'No such directory: %s' % path)
         
         model = {}
         if type:
             model['type'] = type
         
         if ext == '.ipynb':
             model.setdefault('type', 'notebook')
         else:
             model.setdefault('type', 'file')
         
         if model['type'] == 'directory':
             untitled = self.untitled_directory
         elif model['type'] == 'notebook':
             untitled = self.untitled_notebook
             ext = '.ipynb'
         elif model['type'] == 'file':
             untitled = self.untitled_file
         else:
             raise HTTPError(400, "Unexpected model type: %r" % model['type'])
         
         name = self.increment_filename(untitled + ext, path)
         path = u'{0}/{1}'.format(path, name)
         return self.new(model, path)
     
     def new(self, model=None, path=''):
         """Create a new file or directory and return its model with no content.
         
         To create a new untitled entity in a directory, use `new_untitled`.
         """
         path = path.strip('/')
         if model is None:
             model = {}
         
         if path.endswith('.ipynb'):
             model.setdefault('type', 'notebook')
         else:
             model.setdefault('type', 'file')
         
         # no content, not a directory, so fill out new-file model
         if 'content' not in model and model['type'] != 'directory':
             if model['type'] == 'notebook':
                 model['content'] = new_notebook()
                 model['format'] = 'json'
             else:
                 model['content'] = ''
                 model['type'] = 'file'
                 model['format'] = 'text'
         
         model = self.save(model, path)
         return model
 
     def copy(self, from_path, to_path=None):
         """Copy an existing file and return its new model.
 
         If to_path not specified, it will be the parent directory of from_path.
         If to_path is a directory, filename will increment `from_path-Copy#.ext`.
 
         from_path must be a full path to a file.
         """
         path = from_path.strip('/')
         if '/' in path:
             from_dir, from_name = path.rsplit('/', 1)
         else:
             from_dir = ''
             from_name = path
         
         model = self.get_model(path)
         model.pop('path', None)
         model.pop('name', None)
         if model['type'] == 'directory':
             raise HTTPError(400, "Can't copy directories")
         
         if not to_path:
             to_path = from_dir
         if self.dir_exists(to_path):
             base, ext = os.path.splitext(from_name)
             copy_name = u'{0}-Copy{1}'.format(base, ext)
             to_name = self.increment_filename(copy_name, to_path)
             to_path = u'{0}/{1}'.format(to_path, to_name)
         
         model = self.save(model, to_path)
         return model
 
     def log_info(self):
         self.log.info(self.info_string())
 
     def trust_notebook(self, path):
         """Explicitly trust a notebook
 
         Parameters
         ----------
         path : string
             The path of a notebook
         """
         model = self.get_model(path)
         nb = model['content']
         self.log.warn("Trusting notebook %s", path)
         self.notary.mark_cells(nb, True)
         self.save(model, path)
 
     def check_and_sign(self, nb, path=''):
         """Check for trusted cells, and sign the notebook.
 
         Called as a part of saving notebooks.
 
         Parameters
         ----------
         nb : dict
             The notebook dict
         path : string
             The notebook's path (for logging)
         """
         if self.notary.check_cells(nb):
             self.notary.sign(nb)
         else:
             self.log.warn("Saving untrusted notebook %s", path)
 
     def mark_trusted_cells(self, nb, path=''):
         """Mark cells as trusted if the notebook signature matches.
 
         Called as a part of loading notebooks.
 
         Parameters
         ----------
         nb : dict
             The notebook object (in current nbformat)
         path : string
             The notebook's path (for logging)
         """
         trusted = self.notary.check_signature(nb)
         if not trusted:
             self.log.warn("Notebook %s is not trusted", path)
         self.notary.mark_cells(nb, trusted)
 
     def should_list(self, name):
         """Should this file/directory name be displayed in a listing?"""
         return not any(fnmatch(name, glob) for glob in self.hide_globs)

IPython/html/services/contents/tests/test_contents_api.py

@@ -1,504 +1,513 b''
 # coding: utf-8
 """Test the contents webservice API."""
 
 import base64
 import io
 import json
 import os
 import shutil
 from unicodedata import normalize
 
 pjoin = os.path.join
 
 import requests
 
 from IPython.html.utils import url_path_join, url_escape
 from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error
 from IPython.nbformat import read, write, from_dict
 from IPython.nbformat.v4 import (
     new_notebook, new_markdown_cell,
 )
 from IPython.nbformat import v2
 from IPython.utils import py3compat
 from IPython.utils.data import uniq_stable
 
 
 def notebooks_only(dir_model):
     return [nb for nb in dir_model['content'] if nb['type']=='notebook']
 
 def dirs_only(dir_model):
     return [x for x in dir_model['content'] if x['type']=='directory']
 
 
 class API(object):
     """Wrapper for contents API calls."""
     def __init__(self, base_url):
         self.base_url = base_url
 
     def _req(self, verb, path, body=None):
         response = requests.request(verb,
                 url_path_join(self.base_url, 'api/contents', path),
                 data=body,
         )
         response.raise_for_status()
         return response
 
     def list(self, path='/'):
         return self._req('GET', path)
 
-    def read(self, path, type_=None):
+    def read(self, path, type_=None, format=None):
+        query = []
         if type_ is not None:
-            path += '?type=' + type_
+            query.append('type=' + type_)
+        if format is not None:
+            query.append('format=' + format)
+        if query:
+            path += '?' + '&'.join(query)
         return self._req('GET', path)
 
     def create_untitled(self, path='/', ext='.ipynb'):
         body = None
         if ext:
             body = json.dumps({'ext': ext})
         return self._req('POST', path, body)
 
     def mkdir_untitled(self, path='/'):
         return self._req('POST', path, json.dumps({'type': 'directory'}))
 
     def copy(self, copy_from, path='/'):
         body = json.dumps({'copy_from':copy_from})
         return self._req('POST', path, body)
 
     def create(self, path='/'):
         return self._req('PUT', path)
 
     def upload(self, path, body):
         return self._req('PUT', path, body)
 
     def mkdir_untitled(self, path='/'):
         return self._req('POST', path, json.dumps({'type': 'directory'}))
 
     def mkdir(self, path='/'):
         return self._req('PUT', path, json.dumps({'type': 'directory'}))
 
     def copy_put(self, copy_from, path='/'):
         body = json.dumps({'copy_from':copy_from})
         return self._req('PUT', path, body)
 
     def save(self, path, body):
         return self._req('PUT', path, body)
 
     def delete(self, path='/'):
         return self._req('DELETE', path)
 
     def rename(self, path, new_path):
         body = json.dumps({'path': new_path})
         return self._req('PATCH', path, body)
 
     def get_checkpoints(self,  path):
         return self._req('GET', url_path_join(path, 'checkpoints'))
 
     def new_checkpoint(self,  path):
         return self._req('POST', url_path_join(path, 'checkpoints'))
 
     def restore_checkpoint(self, path, checkpoint_id):
         return self._req('POST', url_path_join(path, 'checkpoints', checkpoint_id))
 
     def delete_checkpoint(self, path, checkpoint_id):
         return self._req('DELETE', url_path_join(path, 'checkpoints', checkpoint_id))
 
 class APITest(NotebookTestBase):
     """Test the kernels web service API"""
     dirs_nbs = [('', 'inroot'),
                 ('Directory with spaces in', 'inspace'),
                 (u'unicodé', 'innonascii'),
                 ('foo', 'a'),
                 ('foo', 'b'),
                 ('foo', 'name with spaces'),
                 ('foo', u'unicodé'),
                 ('foo/bar', 'baz'),
                 ('ordering', 'A'),
                 ('ordering', 'b'),
                 ('ordering', 'C'),
                 (u'å b', u'ç d'),
                ]
     hidden_dirs = ['.hidden', '__pycache__']
 
     dirs = uniq_stable([py3compat.cast_unicode(d) for (d,n) in dirs_nbs])
     del dirs[0]  # remove ''
     top_level_dirs = {normalize('NFC', d.split('/')[0]) for d in dirs}
 
     @staticmethod
     def _blob_for_name(name):
         return name.encode('utf-8') + b'\xFF'
 
     @staticmethod
     def _txt_for_name(name):
         return u'%s text file' % name
 
     def setUp(self):
         nbdir = self.notebook_dir.name
         self.blob = os.urandom(100)
         self.b64_blob = base64.encodestring(self.blob).decode('ascii')
 
         for d in (self.dirs + self.hidden_dirs):
             d.replace('/', os.sep)
             if not os.path.isdir(pjoin(nbdir, d)):
                 os.mkdir(pjoin(nbdir, d))
 
         for d, name in self.dirs_nbs:
             d = d.replace('/', os.sep)
             # create a notebook
             with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w',
                          encoding='utf-8') as f:
                 nb = new_notebook()
                 write(nb, f, version=4)
 
             # create a text file
             with io.open(pjoin(nbdir, d, '%s.txt' % name), 'w',
                          encoding='utf-8') as f:
                 f.write(self._txt_for_name(name))
 
             # create a binary file
             with io.open(pjoin(nbdir, d, '%s.blob' % name), 'wb') as f:
                 f.write(self._blob_for_name(name))
 
         self.api = API(self.base_url())
 
     def tearDown(self):
         nbdir = self.notebook_dir.name
 
         for dname in (list(self.top_level_dirs) + self.hidden_dirs):
             shutil.rmtree(pjoin(nbdir, dname), ignore_errors=True)
 
         if os.path.isfile(pjoin(nbdir, 'inroot.ipynb')):
             os.unlink(pjoin(nbdir, 'inroot.ipynb'))
 
     def test_list_notebooks(self):
         nbs = notebooks_only(self.api.list().json())
         self.assertEqual(len(nbs), 1)
         self.assertEqual(nbs[0]['name'], 'inroot.ipynb')
 
         nbs = notebooks_only(self.api.list('/Directory with spaces in/').json())
         self.assertEqual(len(nbs), 1)
         self.assertEqual(nbs[0]['name'], 'inspace.ipynb')
 
         nbs = notebooks_only(self.api.list(u'/unicodé/').json())
         self.assertEqual(len(nbs), 1)
         self.assertEqual(nbs[0]['name'], 'innonascii.ipynb')
         self.assertEqual(nbs[0]['path'], u'unicodé/innonascii.ipynb')
 
         nbs = notebooks_only(self.api.list('/foo/bar/').json())
         self.assertEqual(len(nbs), 1)
         self.assertEqual(nbs[0]['name'], 'baz.ipynb')
         self.assertEqual(nbs[0]['path'], 'foo/bar/baz.ipynb')
 
         nbs = notebooks_only(self.api.list('foo').json())
         self.assertEqual(len(nbs), 4)
         nbnames = { normalize('NFC', n['name']) for n in nbs }
         expected = [ u'a.ipynb', u'b.ipynb', u'name with spaces.ipynb', u'unicodé.ipynb']
         expected = { normalize('NFC', name) for name in expected }
         self.assertEqual(nbnames, expected)
 
         nbs = notebooks_only(self.api.list('ordering').json())
         nbnames = [n['name'] for n in nbs]
         expected = ['A.ipynb', 'b.ipynb', 'C.ipynb']
         self.assertEqual(nbnames, expected)
 
     def test_list_dirs(self):
         print(self.api.list().json())
         dirs = dirs_only(self.api.list().json())
         dir_names = {normalize('NFC', d['name']) for d in dirs}
         print(dir_names)
         print(self.top_level_dirs)
         self.assertEqual(dir_names, self.top_level_dirs)  # Excluding hidden dirs
 
     def test_list_nonexistant_dir(self):
         with assert_http_error(404):
             self.api.list('nonexistant')
 
     def test_get_nb_contents(self):
         for d, name in self.dirs_nbs:
             path = url_path_join(d, name + '.ipynb')
             nb = self.api.read(path).json()
             self.assertEqual(nb['name'], u'%s.ipynb' % name)
             self.assertEqual(nb['path'], path)
             self.assertEqual(nb['type'], 'notebook')
             self.assertIn('content', nb)
             self.assertEqual(nb['format'], 'json')
             self.assertIn('content', nb)
             self.assertIn('metadata', nb['content'])
             self.assertIsInstance(nb['content']['metadata'], dict)
 
     def test_get_contents_no_such_file(self):
         # Name that doesn't exist - should be a 404
         with assert_http_error(404):
             self.api.read('foo/q.ipynb')
 
     def test_get_text_file_contents(self):
         for d, name in self.dirs_nbs:
             path = url_path_join(d, name + '.txt')
             model = self.api.read(path).json()
             self.assertEqual(model['name'], u'%s.txt' % name)
             self.assertEqual(model['path'], path)
             self.assertIn('content', model)
             self.assertEqual(model['format'], 'text')
             self.assertEqual(model['type'], 'file')
             self.assertEqual(model['content'], self._txt_for_name(name))
 
         # Name that doesn't exist - should be a 404
         with assert_http_error(404):
             self.api.read('foo/q.txt')
 
+        # Specifying format=text should fail on a non-UTF-8 file
+        with assert_http_error(400):
+            self.api.read('foo/bar/baz.blob', type_='file', format='text')
+
     def test_get_binary_file_contents(self):
         for d, name in self.dirs_nbs:
             path = url_path_join(d, name + '.blob')
             model = self.api.read(path).json()
             self.assertEqual(model['name'], u'%s.blob' % name)
             self.assertEqual(model['path'], path)
             self.assertIn('content', model)
             self.assertEqual(model['format'], 'base64')
             self.assertEqual(model['type'], 'file')
             b64_data = base64.encodestring(self._blob_for_name(name)).decode('ascii')
             self.assertEqual(model['content'], b64_data)
 
         # Name that doesn't exist - should be a 404
         with assert_http_error(404):
             self.api.read('foo/q.txt')
 
     def test_get_bad_type(self):
         with assert_http_error(400):
             self.api.read(u'unicodé', type_='file')  # this is a directory
 
         with assert_http_error(400):
             self.api.read(u'unicodé/innonascii.ipynb', type_='directory')
 
     def _check_created(self, resp, path, type='notebook'):
         self.assertEqual(resp.status_code, 201)
         location_header = py3compat.str_to_unicode(resp.headers['Location'])
         self.assertEqual(location_header, url_escape(url_path_join(u'/api/contents', path)))
         rjson = resp.json()
         self.assertEqual(rjson['name'], path.rsplit('/', 1)[-1])
         self.assertEqual(rjson['path'], path)
         self.assertEqual(rjson['type'], type)
         isright = os.path.isdir if type == 'directory' else os.path.isfile
         assert isright(pjoin(
             self.notebook_dir.name,
             path.replace('/', os.sep),
         ))
 
     def test_create_untitled(self):
         resp = self.api.create_untitled(path=u'å b')
         self._check_created(resp, u'å b/Untitled0.ipynb')
 
         # Second time
         resp = self.api.create_untitled(path=u'å b')
         self._check_created(resp, u'å b/Untitled1.ipynb')
 
         # And two directories down
         resp = self.api.create_untitled(path='foo/bar')
         self._check_created(resp, 'foo/bar/Untitled0.ipynb')
 
     def test_create_untitled_txt(self):
         resp = self.api.create_untitled(path='foo/bar', ext='.txt')
         self._check_created(resp, 'foo/bar/untitled0.txt', type='file')
 
         resp = self.api.read(path='foo/bar/untitled0.txt')
         model = resp.json()
         self.assertEqual(model['type'], 'file')
         self.assertEqual(model['format'], 'text')
         self.assertEqual(model['content'], '')
 
     def test_upload(self):
         nb = new_notebook()
         nbmodel = {'content': nb, 'type': 'notebook'}
         path = u'å b/Upload tést.ipynb'
         resp = self.api.upload(path, body=json.dumps(nbmodel))
         self._check_created(resp, path)
 
     def test_mkdir_untitled(self):
         resp = self.api.mkdir_untitled(path=u'å b')
         self._check_created(resp, u'å b/Untitled Folder0', type='directory')
 
         # Second time
         resp = self.api.mkdir_untitled(path=u'å b')
         self._check_created(resp, u'å b/Untitled Folder1', type='directory')
 
         # And two directories down
         resp = self.api.mkdir_untitled(path='foo/bar')
         self._check_created(resp, 'foo/bar/Untitled Folder0', type='directory')
 
     def test_mkdir(self):
         path = u'å b/New ∂ir'
         resp = self.api.mkdir(path)
         self._check_created(resp, path, type='directory')
 
     def test_mkdir_hidden_400(self):
         with assert_http_error(400):
             resp = self.api.mkdir(u'å b/.hidden')
 
     def test_upload_txt(self):
         body = u'ünicode téxt'
         model = {
             'content' : body,
             'format'  : 'text',
             'type'    : 'file',
         }
         path = u'å b/Upload tést.txt'
         resp = self.api.upload(path, body=json.dumps(model))
 
         # check roundtrip
         resp = self.api.read(path)
         model = resp.json()
         self.assertEqual(model['type'], 'file')
         self.assertEqual(model['format'], 'text')
         self.assertEqual(model['content'], body)
 
     def test_upload_b64(self):
         body = b'\xFFblob'
         b64body = base64.encodestring(body).decode('ascii')
         model = {
             'content' : b64body,
             'format'  : 'base64',
             'type'    : 'file',
         }
         path = u'å b/Upload tést.blob'
         resp = self.api.upload(path, body=json.dumps(model))
 
         # check roundtrip
         resp = self.api.read(path)
         model = resp.json()
         self.assertEqual(model['type'], 'file')
         self.assertEqual(model['path'], path)
         self.assertEqual(model['format'], 'base64')
         decoded = base64.decodestring(model['content'].encode('ascii'))
         self.assertEqual(decoded, body)
 
     def test_upload_v2(self):
         nb = v2.new_notebook()
         ws = v2.new_worksheet()
         nb.worksheets.append(ws)
         ws.cells.append(v2.new_code_cell(input='print("hi")'))
         nbmodel = {'content': nb, 'type': 'notebook'}
         path = u'å b/Upload tést.ipynb'
         resp = self.api.upload(path, body=json.dumps(nbmodel))
         self._check_created(resp, path)
         resp = self.api.read(path)
         data = resp.json()
         self.assertEqual(data['content']['nbformat'], 4)
 
     def test_copy(self):
         resp = self.api.copy(u'å b/ç d.ipynb', u'unicodé')
         self._check_created(resp, u'unicodé/ç d-Copy0.ipynb')
         
         resp = self.api.copy(u'å b/ç d.ipynb', u'å b')
         self._check_created(resp, u'å b/ç d-Copy0.ipynb')
 
     def test_copy_path(self):
         resp = self.api.copy(u'foo/a.ipynb', u'å b')
         self._check_created(resp, u'å b/a-Copy0.ipynb')
 
     def test_copy_put_400(self):
         with assert_http_error(400):
             resp = self.api.copy_put(u'å b/ç d.ipynb', u'å b/cøpy.ipynb')
 
     def test_copy_dir_400(self):
         # can't copy directories
         with assert_http_error(400):
             resp = self.api.copy(u'å b', u'foo')
 
     def test_delete(self):
         for d, name in self.dirs_nbs:
             print('%r, %r' % (d, name))
             resp = self.api.delete(url_path_join(d, name + '.ipynb'))
             self.assertEqual(resp.status_code, 204)
 
         for d in self.dirs + ['/']:
             nbs = notebooks_only(self.api.list(d).json())
             print('------')
             print(d)
             print(nbs)
             self.assertEqual(nbs, [])
 
     def test_delete_dirs(self):
         # depth-first delete everything, so we don't try to delete empty directories
         for name in sorted(self.dirs + ['/'], key=len, reverse=True):
             listing = self.api.list(name).json()['content']
             for model in listing:
                 self.api.delete(model['path'])
         listing = self.api.list('/').json()['content']
         self.assertEqual(listing, [])
 
     def test_delete_non_empty_dir(self):
         """delete non-empty dir raises 400"""
         with assert_http_error(400):
             self.api.delete(u'å b')
 
     def test_rename(self):
         resp = self.api.rename('foo/a.ipynb', 'foo/z.ipynb')
         self.assertEqual(resp.headers['Location'].split('/')[-1], 'z.ipynb')
         self.assertEqual(resp.json()['name'], 'z.ipynb')
         self.assertEqual(resp.json()['path'], 'foo/z.ipynb')
         assert os.path.isfile(pjoin(self.notebook_dir.name, 'foo', 'z.ipynb'))
 
         nbs = notebooks_only(self.api.list('foo').json())
         nbnames = set(n['name'] for n in nbs)
         self.assertIn('z.ipynb', nbnames)
         self.assertNotIn('a.ipynb', nbnames)
 
     def test_rename_existing(self):
         with assert_http_error(409):
             self.api.rename('foo/a.ipynb', 'foo/b.ipynb')
 
     def test_save(self):
         resp = self.api.read('foo/a.ipynb')
         nbcontent = json.loads(resp.text)['content']
         nb = from_dict(nbcontent)
         nb.cells.append(new_markdown_cell(u'Created by test ³'))
 
         nbmodel= {'content': nb, 'type': 'notebook'}
         resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
 
         nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')
         with io.open(nbfile, 'r', encoding='utf-8') as f:
             newnb = read(f, as_version=4)
         self.assertEqual(newnb.cells[0].source,
                          u'Created by test ³')
         nbcontent = self.api.read('foo/a.ipynb').json()['content']
         newnb = from_dict(nbcontent)
         self.assertEqual(newnb.cells[0].source,
                          u'Created by test ³')
 
 
     def test_checkpoints(self):
         resp = self.api.read('foo/a.ipynb')
         r = self.api.new_checkpoint('foo/a.ipynb')
         self.assertEqual(r.status_code, 201)
         cp1 = r.json()
         self.assertEqual(set(cp1), {'id', 'last_modified'})
         self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id'])
 
         # Modify it
         nbcontent = json.loads(resp.text)['content']
         nb = from_dict(nbcontent)
         hcell = new_markdown_cell('Created by test')
         nb.cells.append(hcell)
         # Save
         nbmodel= {'content': nb, 'type': 'notebook'}
         resp = self.api.save('foo/a.ipynb', body=json.dumps(nbmodel))
 
         # List checkpoints
         cps = self.api.get_checkpoints('foo/a.ipynb').json()
         self.assertEqual(cps, [cp1])
 
         nbcontent = self.api.read('foo/a.ipynb').json()['content']
         nb = from_dict(nbcontent)
         self.assertEqual(nb.cells[0].source, 'Created by test')
 
         # Restore cp1
         r = self.api.restore_checkpoint('foo/a.ipynb', cp1['id'])
         self.assertEqual(r.status_code, 204)
         nbcontent = self.api.read('foo/a.ipynb').json()['content']
         nb = from_dict(nbcontent)
         self.assertEqual(nb.cells, [])
 
         # Delete cp1
         r = self.api.delete_checkpoint('foo/a.ipynb', cp1['id'])
         self.assertEqual(r.status_code, 204)
         cps = self.api.get_checkpoints('foo/a.ipynb').json()
         self.assertEqual(cps, [])

IPython/html/services/contents/tests/test_manager.py

@@ -1,362 +1,365 b''
 # coding: utf-8
 """Tests for the notebook manager."""
 from __future__ import print_function
 
 import logging
 import os
 
 from tornado.web import HTTPError
 from unittest import TestCase
 from tempfile import NamedTemporaryFile
 
 from IPython.nbformat import v4 as nbformat
 
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.traitlets import TraitError
 from IPython.html.utils import url_path_join
 from IPython.testing import decorators as dec
 
 from ..filemanager import FileContentsManager
 from ..manager import ContentsManager
 
 
 class TestFileContentsManager(TestCase):
 
     def test_root_dir(self):
         with TemporaryDirectory() as td:
             fm = FileContentsManager(root_dir=td)
             self.assertEqual(fm.root_dir, td)
 
     def test_missing_root_dir(self):
         with TemporaryDirectory() as td:
             root = os.path.join(td, 'notebook', 'dir', 'is', 'missing')
             self.assertRaises(TraitError, FileContentsManager, root_dir=root)
 
     def test_invalid_root_dir(self):
         with NamedTemporaryFile() as tf:
             self.assertRaises(TraitError, FileContentsManager, root_dir=tf.name)
 
     def test_get_os_path(self):
         # full filesystem path should be returned with correct operating system
         # separators.
         with TemporaryDirectory() as td:
             root = td
             fm = FileContentsManager(root_dir=root)
             path = fm._get_os_path('/path/to/notebook/test.ipynb')
             rel_path_list =  '/path/to/notebook/test.ipynb'.split('/')
             fs_path = os.path.join(fm.root_dir, *rel_path_list)
             self.assertEqual(path, fs_path)
 
             fm = FileContentsManager(root_dir=root)
             path = fm._get_os_path('test.ipynb')
             fs_path = os.path.join(fm.root_dir, 'test.ipynb')
             self.assertEqual(path, fs_path)
 
             fm = FileContentsManager(root_dir=root)
             path = fm._get_os_path('////test.ipynb')
             fs_path = os.path.join(fm.root_dir, 'test.ipynb')
             self.assertEqual(path, fs_path)
 
     def test_checkpoint_subdir(self):
         subd = u'sub ∂ir'
         cp_name = 'test-cp.ipynb'
         with TemporaryDirectory() as td:
             root = td
             os.mkdir(os.path.join(td, subd))
             fm = FileContentsManager(root_dir=root)
             cp_dir = fm.get_checkpoint_path('cp', 'test.ipynb')
             cp_subdir = fm.get_checkpoint_path('cp', '/%s/test.ipynb' % subd)
         self.assertNotEqual(cp_dir, cp_subdir)
         self.assertEqual(cp_dir, os.path.join(root, fm.checkpoint_dir, cp_name))
         self.assertEqual(cp_subdir, os.path.join(root, subd, fm.checkpoint_dir, cp_name))
 
 
 class TestContentsManager(TestCase):
 
     def setUp(self):
         self._temp_dir = TemporaryDirectory()
         self.td = self._temp_dir.name
         self.contents_manager = FileContentsManager(
             root_dir=self.td,
             log=logging.getLogger()
         )
 
     def tearDown(self):
         self._temp_dir.cleanup()
 
     def make_dir(self, abs_path, rel_path):
         """make subdirectory, rel_path is the relative path
         to that directory from the location where the server started"""
         os_path = os.path.join(abs_path, rel_path)
         try:
             os.makedirs(os_path)
         except OSError:
             print("Directory already exists: %r" % os_path)
         return os_path
 
     def add_code_cell(self, nb):
         output = nbformat.new_output("display_data", {'application/javascript': "alert('hi');"})
         cell = nbformat.new_code_cell("print('hi')", outputs=[output])
         nb.cells.append(cell)
 
     def new_notebook(self):
         cm = self.contents_manager
         model = cm.new_untitled(type='notebook')
         name = model['name']
         path = model['path']
 
         full_model = cm.get_model(path)
         nb = full_model['content']
         self.add_code_cell(nb)
 
         cm.save(full_model, path)
         return nb, name, path
 
     def test_new_untitled(self):
         cm = self.contents_manager
         # Test in root directory
         model = cm.new_untitled(type='notebook')
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertIn('type', model)
         self.assertEqual(model['type'], 'notebook')
         self.assertEqual(model['name'], 'Untitled0.ipynb')
         self.assertEqual(model['path'], 'Untitled0.ipynb')
 
         # Test in sub-directory
         model = cm.new_untitled(type='directory')
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertIn('type', model)
         self.assertEqual(model['type'], 'directory')
         self.assertEqual(model['name'], 'Untitled Folder0')
         self.assertEqual(model['path'], 'Untitled Folder0')
         sub_dir = model['path']
         
         model = cm.new_untitled(path=sub_dir)
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertIn('type', model)
         self.assertEqual(model['type'], 'file')
         self.assertEqual(model['name'], 'untitled0')
         self.assertEqual(model['path'], '%s/untitled0' % sub_dir)
 
     def test_get(self):
         cm = self.contents_manager
         # Create a notebook
         model = cm.new_untitled(type='notebook')
         name = model['name']
         path = model['path']
 
         # Check that we 'get' on the notebook we just created
         model2 = cm.get_model(path)
         assert isinstance(model2, dict)
         self.assertIn('name', model2)
         self.assertIn('path', model2)
         self.assertEqual(model['name'], name)
         self.assertEqual(model['path'], path)
 
         nb_as_file = cm.get_model(path, content=True, type_='file')
         self.assertEqual(nb_as_file['path'], path)
         self.assertEqual(nb_as_file['type'], 'file')
         self.assertEqual(nb_as_file['format'], 'text')
         self.assertNotIsInstance(nb_as_file['content'], dict)
 
+        nb_as_bin_file = cm.get_model(path, content=True, type_='file', format='base64')
+        self.assertEqual(nb_as_bin_file['format'], 'base64')
+
         # Test in sub-directory
         sub_dir = '/foo/'
         self.make_dir(cm.root_dir, 'foo')
         model = cm.new_untitled(path=sub_dir, ext='.ipynb')
         model2 = cm.get_model(sub_dir + name)
         assert isinstance(model2, dict)
         self.assertIn('name', model2)
         self.assertIn('path', model2)
         self.assertIn('content', model2)
         self.assertEqual(model2['name'], 'Untitled0.ipynb')
         self.assertEqual(model2['path'], '{0}/{1}'.format(sub_dir.strip('/'), name))
 
         # Test getting directory model
         dirmodel = cm.get_model('foo')
         self.assertEqual(dirmodel['type'], 'directory')
 
         with self.assertRaises(HTTPError):
             cm.get_model('foo', type_='file')
 
     
     @dec.skip_win32
     def test_bad_symlink(self):
         cm = self.contents_manager
         path = 'test bad symlink'
         os_path = self.make_dir(cm.root_dir, path)
         
         file_model = cm.new_untitled(path=path, ext='.txt')
         
         # create a broken symlink
         os.symlink("target", os.path.join(os_path, "bad symlink"))
         model = cm.get_model(path)
         self.assertEqual(model['content'], [file_model])
     
     @dec.skip_win32
     def test_good_symlink(self):
         cm = self.contents_manager
         parent = 'test good symlink'
         name = 'good symlink'
         path = '{0}/{1}'.format(parent, name)
         os_path = self.make_dir(cm.root_dir, parent)
         
         file_model = cm.new(path=parent + '/zfoo.txt')
         
         # create a good symlink
         os.symlink(file_model['name'], os.path.join(os_path, name))
         symlink_model = cm.get_model(path, content=False)
         dir_model = cm.get_model(parent)
         self.assertEqual(
             sorted(dir_model['content'], key=lambda x: x['name']),
             [symlink_model, file_model],
         )
     
     def test_update(self):
         cm = self.contents_manager
         # Create a notebook
         model = cm.new_untitled(type='notebook')
         name = model['name']
         path = model['path']
 
         # Change the name in the model for rename
         model['path'] = 'test.ipynb'
         model = cm.update(model, path)
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertEqual(model['name'], 'test.ipynb')
 
         # Make sure the old name is gone
         self.assertRaises(HTTPError, cm.get_model, path)
 
         # Test in sub-directory
         # Create a directory and notebook in that directory
         sub_dir = '/foo/'
         self.make_dir(cm.root_dir, 'foo')
         model = cm.new_untitled(path=sub_dir, type='notebook')
         name = model['name']
         path = model['path']
 
         # Change the name in the model for rename
         d = path.rsplit('/', 1)[0]
         new_path = model['path'] = d + '/test_in_sub.ipynb'
         model = cm.update(model, path)
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertEqual(model['name'], 'test_in_sub.ipynb')
         self.assertEqual(model['path'], new_path)
 
         # Make sure the old name is gone
         self.assertRaises(HTTPError, cm.get_model, path)
 
     def test_save(self):
         cm = self.contents_manager
         # Create a notebook
         model = cm.new_untitled(type='notebook')
         name = model['name']
         path = model['path']
 
         # Get the model with 'content'
         full_model = cm.get_model(path)
 
         # Save the notebook
         model = cm.save(full_model, path)
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertEqual(model['name'], name)
         self.assertEqual(model['path'], path)
 
         # Test in sub-directory
         # Create a directory and notebook in that directory
         sub_dir = '/foo/'
         self.make_dir(cm.root_dir, 'foo')
         model = cm.new_untitled(path=sub_dir, type='notebook')
         name = model['name']
         path = model['path']
         model = cm.get_model(path)
 
         # Change the name in the model for rename
         model = cm.save(model, path)
         assert isinstance(model, dict)
         self.assertIn('name', model)
         self.assertIn('path', model)
         self.assertEqual(model['name'], 'Untitled0.ipynb')
         self.assertEqual(model['path'], 'foo/Untitled0.ipynb')
 
     def test_delete(self):
         cm = self.contents_manager
         # Create a notebook
         nb, name, path = self.new_notebook()
 
         # Delete the notebook
         cm.delete(path)
 
         # Check that a 'get' on the deleted notebook raises and error
         self.assertRaises(HTTPError, cm.get_model, path)
 
     def test_copy(self):
         cm = self.contents_manager
         parent = u'å b'
         name = u'nb √.ipynb'
         path = u'{0}/{1}'.format(parent, name)
         os.mkdir(os.path.join(cm.root_dir, parent))
         orig = cm.new(path=path)
 
         # copy with unspecified name
         copy = cm.copy(path)
         self.assertEqual(copy['name'], orig['name'].replace('.ipynb', '-Copy0.ipynb'))
 
         # copy with specified name
         copy2 = cm.copy(path, u'å b/copy 2.ipynb')
         self.assertEqual(copy2['name'], u'copy 2.ipynb')
         self.assertEqual(copy2['path'], u'å b/copy 2.ipynb')
 
     def test_trust_notebook(self):
         cm = self.contents_manager
         nb, name, path = self.new_notebook()
 
         untrusted = cm.get_model(path)['content']
         assert not cm.notary.check_cells(untrusted)
 
         # print(untrusted)
         cm.trust_notebook(path)
         trusted = cm.get_model(path)['content']
         # print(trusted)
         assert cm.notary.check_cells(trusted)
 
     def test_mark_trusted_cells(self):
         cm = self.contents_manager
         nb, name, path = self.new_notebook()
 
         cm.mark_trusted_cells(nb, path)
         for cell in nb.cells:
             if cell.cell_type == 'code':
                 assert not cell.metadata.trusted
 
         cm.trust_notebook(path)
         nb = cm.get_model(path)['content']
         for cell in nb.cells:
             if cell.cell_type == 'code':
                 assert cell.metadata.trusted
 
     def test_check_and_sign(self):
         cm = self.contents_manager
         nb, name, path = self.new_notebook()
 
         cm.mark_trusted_cells(nb, path)
         cm.check_and_sign(nb, path)
         assert not cm.notary.check_signature(nb)
 
         cm.trust_notebook(path)
         nb = cm.get_model(path)['content']
         cm.mark_trusted_cells(nb, path)
         cm.check_and_sign(nb, path)
         assert cm.notary.check_signature(nb)