manager.py
468 lines
| 14.5 KiB
| text/x-python
|
PythonLexer
|
r17524 | """A base class for contents managers.""" | ||
|
r8180 | |||
|
r17524 | # Copyright (c) IPython Development Team. | ||
# Distributed under the terms of the Modified BSD License. | ||||
|
r8180 | |||
|
r15525 | from fnmatch import fnmatch | ||
|
r15292 | import itertools | ||
|
r18249 | import json | ||
|
r8181 | import os | ||
|
r18813 | import re | ||
|
r8180 | |||
|
r17530 | from tornado.web import HTTPError | ||
|
r19839 | from .checkpoints import Checkpoints | ||
|
r8180 | from IPython.config.configurable import LoggingConfigurable | ||
|
r18607 | from IPython.nbformat import sign, validate, ValidationError | ||
from IPython.nbformat.v4 import new_notebook | ||||
|
r19299 | from IPython.utils.importstring import import_item | ||
|
r19727 | from IPython.utils.traitlets import ( | ||
Any, | ||||
Dict, | ||||
Instance, | ||||
List, | ||||
TraitError, | ||||
Type, | ||||
Unicode, | ||||
) | ||||
|
r19299 | from IPython.utils.py3compat import string_types | ||
|
r8180 | |||
|
r18813 | copy_pat = re.compile(r'\-Copy\d*\.') | ||
|
r8180 | |||
|
r19727 | |||
|
r17524 | class ContentsManager(LoggingConfigurable): | ||
|
r17535 | """Base class for serving files and directories. | ||
This serves any text or binary file, | ||||
as well as directories, | ||||
with special handling for JSON notebook documents. | ||||
Most APIs take a path argument, | ||||
which is always an API-style unicode path, | ||||
and always refers to a directory. | ||||
- unicode, not url-escaped | ||||
- '/'-separated | ||||
- leading and trailing '/' will be stripped | ||||
- if unspecified, path defaults to '', | ||||
indicating the root path. | ||||
""" | ||||
|
r17523 | |||
|
r14857 | notary = Instance(sign.NotebookNotary) | ||
def _notary_default(self): | ||||
return sign.NotebookNotary(parent=self) | ||||
|
r17523 | |||
|
r17529 | hide_globs = List(Unicode, [ | ||
u'__pycache__', '*.pyc', '*.pyo', | ||||
'.DS_Store', '*.so', '*.dylib', '*~', | ||||
], config=True, help=""" | ||||
|
r15525 | Glob patterns to hide in file and directory listings. | ||
""") | ||||
|
r17535 | untitled_notebook = Unicode("Untitled", config=True, | ||
help="The base name used when creating untitled notebooks." | ||||
) | ||||
untitled_file = Unicode("untitled", config=True, | ||||
help="The base name used when creating untitled files." | ||||
) | ||||
untitled_directory = Unicode("Untitled Folder", config=True, | ||||
help="The base name used when creating untitled directories." | ||||
) | ||||
|
r19299 | pre_save_hook = Any(None, config=True, | ||
help="""Python callable or importstring thereof | ||||
To be called on a contents model prior to save. | ||||
This can be used to process the structure, | ||||
such as removing notebook outputs or other side effects that | ||||
should not be saved. | ||||
|
r20502 | It will be called as (all arguments passed by keyword):: | ||
|
r19299 | |||
hook(path=path, model=model, contents_manager=self) | ||||
|
r20502 | - model: the model to be saved. Includes file contents. | ||
Modifying this dict will affect the file that is stored. | ||||
- path: the API path of the save destination | ||||
- contents_manager: this ContentsManager instance | ||||
|
r19299 | """ | ||
) | ||||
def _pre_save_hook_changed(self, name, old, new): | ||||
if new and isinstance(new, string_types): | ||||
self.pre_save_hook = import_item(self.pre_save_hook) | ||||
elif new: | ||||
if not callable(new): | ||||
raise TraitError("pre_save_hook must be callable") | ||||
def run_pre_save_hook(self, model, path, **kwargs): | ||||
"""Run the pre-save hook if defined, and log errors""" | ||||
if self.pre_save_hook: | ||||
try: | ||||
self.log.debug("Running pre-save hook on %s", path) | ||||
self.pre_save_hook(model=model, path=path, contents_manager=self, **kwargs) | ||||
except Exception: | ||||
self.log.error("Pre-save hook failed on %s", path, exc_info=True) | ||||
|
r19839 | checkpoints_class = Type(Checkpoints, config=True) | ||
checkpoints = Instance(Checkpoints, config=True) | ||||
|
r20483 | checkpoints_kwargs = Dict(config=True) | ||
|
r19727 | |||
|
r19839 | def _checkpoints_default(self): | ||
return self.checkpoints_class(**self.checkpoints_kwargs) | ||||
|
r19727 | |||
|
r19839 | def _checkpoints_kwargs_default(self): | ||
|
r19727 | return dict( | ||
parent=self, | ||||
log=self.log, | ||||
) | ||||
|
r17524 | # ContentsManager API part 1: methods that must be | ||
|
r15293 | # implemented in subclasses. | ||
|
r18749 | def dir_exists(self, path): | ||
|
r13067 | """Does the API-style path (directory) actually exist? | ||
|
r17523 | |||
|
r17535 | Like os.path.isdir | ||
|
r13070 | Override this method in subclasses. | ||
|
r17523 | |||
|
r13026 | Parameters | ||
---------- | ||||
|
r13067 | path : string | ||
|
r15664 | The path to check | ||
|
r17523 | |||
|
r13026 | Returns | ||
------- | ||||
|
r13067 | exists : bool | ||
Whether the path does indeed exist. | ||||
|
r13026 | """ | ||
|
r13070 | raise NotImplementedError | ||
|
r15097 | |||
def is_hidden(self, path): | ||||
"""Does the API style path correspond to a hidden directory or file? | ||||
|
r17523 | |||
|
r15097 | Parameters | ||
---------- | ||||
path : string | ||||
The path to check. This is an API path (`/` separated, | ||||
|
r17524 | relative to root dir). | ||
|
r17523 | |||
|
r15097 | Returns | ||
------- | ||||
|
r17535 | hidden : bool | ||
|
r15097 | Whether the path is hidden. | ||
|
r17523 | |||
|
r15097 | """ | ||
raise NotImplementedError | ||||
|
r18749 | def file_exists(self, path=''): | ||
"""Does a file exist at the given path? | ||||
|
r17535 | |||
Like os.path.isfile | ||||
Override this method in subclasses. | ||||
|
r15291 | |||
Parameters | ||||
---------- | ||||
name : string | ||||
|
r17529 | The name of the file you are checking. | ||
|
r15291 | path : string | ||
|
r17529 | The relative path to the file's directory (with '/' as separator) | ||
|
r15291 | |||
Returns | ||||
------- | ||||
|
r17535 | exists : bool | ||
Whether the file exists. | ||||
|
r15291 | """ | ||
raise NotImplementedError('must be implemented in a subclass') | ||||
|
r18749 | def exists(self, path): | ||
|
r18758 | """Does a file or directory exist at the given path? | ||
|
r8180 | |||
|
r17535 | Like os.path.exists | ||
|
r8180 | |||
|
r17535 | Parameters | ||
---------- | ||||
path : string | ||||
The relative path to the file's directory (with '/' as separator) | ||||
|
r8180 | |||
|
r17535 | Returns | ||
------- | ||||
exists : bool | ||||
Whether the target exists. | ||||
|
r8180 | """ | ||
|
r18749 | return self.file_exists(path) or self.dir_exists(path) | ||
|
r8180 | |||
|
r19391 | def get(self, path, content=True, type=None, format=None): | ||
|
r17529 | """Get the model of a file or directory with or without content.""" | ||
|
r8180 | raise NotImplementedError('must be implemented in a subclass') | ||
|
r18749 | def save(self, model, path): | ||
|
r19299 | """Save the file or directory and return the model with no content. | ||
Save implementations should call self.run_pre_save_hook(model=model, path=path) | ||||
prior to writing any data. | ||||
""" | ||||
|
r13046 | raise NotImplementedError('must be implemented in a subclass') | ||
|
r8180 | |||
|
r19727 | def delete_file(self, path): | ||
|
r18749 | """Delete file or directory by path.""" | ||
|
r8180 | raise NotImplementedError('must be implemented in a subclass') | ||
|
r19727 | def rename_file(self, old_path, new_path): | ||
"""Rename a file.""" | ||||
raise NotImplementedError('must be implemented in a subclass') | ||||
|
r17523 | |||
|
r17524 | # ContentsManager API part 2: methods that have useable default | ||
|
r15293 | # implementations, but can be overridden in subclasses. | ||
|
r19727 | def delete(self, path): | ||
"""Delete a file/directory and any associated checkpoints.""" | ||||
self.delete_file(path) | ||||
|
r19839 | self.checkpoints.delete_all_checkpoints(path) | ||
|
r19727 | |||
def rename(self, old_path, new_path): | ||||
"""Rename a file and any checkpoints associated with that file.""" | ||||
self.rename_file(old_path, new_path) | ||||
|
r19839 | self.checkpoints.rename_all_checkpoints(old_path, new_path) | ||
|
r19727 | |||
|
r19712 | def update(self, model, path): | ||
"""Update the file's path | ||||
For use in PATCH requests, to enable renaming a file without | ||||
re-uploading its contents. Only used for renaming at the moment. | ||||
""" | ||||
path = path.strip('/') | ||||
new_path = model.get('path', path).strip('/') | ||||
if path != new_path: | ||||
self.rename(path, new_path) | ||||
model = self.get(new_path, content=False) | ||||
return model | ||||
|
r17535 | def info_string(self): | ||
return "Serving contents" | ||||
|
r18749 | def get_kernel_path(self, path, model=None): | ||
|
r18758 | """Return the API path for the kernel | ||
KernelManagers can turn this value into a filesystem path, | ||||
or ignore it altogether. | ||||
|
r19136 | |||
The default value here will start kernels in the directory of the | ||||
notebook server. FileContentsManager overrides this to use the | ||||
directory containing the notebook. | ||||
|
r18758 | """ | ||
|
r19136 | return '' | ||
|
r16052 | |||
|
r18813 | def increment_filename(self, filename, path='', insert=''): | ||
|
r17524 | """Increment a filename until it is unique. | ||
|
r17523 | |||
|
r15293 | Parameters | ||
---------- | ||||
|
r17524 | filename : unicode | ||
The name of a file, including extension | ||||
|
r15293 | path : unicode | ||
|
r17535 | The API path of the target's directory | ||
|
r15293 | |||
Returns | ||||
------- | ||||
name : unicode | ||||
|
r17524 | A filename that is unique, based on the input filename. | ||
|
r15293 | """ | ||
path = path.strip('/') | ||||
|
r17524 | basename, ext = os.path.splitext(filename) | ||
|
r15293 | for i in itertools.count(): | ||
|
r18813 | if i: | ||
insert_i = '{}{}'.format(insert, i) | ||||
else: | ||||
insert_i = '' | ||||
name = u'{basename}{insert}{ext}'.format(basename=basename, | ||||
insert=insert_i, ext=ext) | ||||
|
r18758 | if not self.exists(u'{}/{}'.format(path, name)): | ||
|
r15293 | break | ||
return name | ||||
|
r18249 | def validate_notebook_model(self, model): | ||
"""Add failed-validation message to model""" | ||||
try: | ||||
|
r18607 | validate(model['content']) | ||
except ValidationError as e: | ||||
|
r18754 | model['message'] = u'Notebook Validation failed: {}:\n{}'.format( | ||
|
r18249 | e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'), | ||
) | ||||
return model | ||||
|
r18759 | |||
def new_untitled(self, path='', type='', ext=''): | ||||
"""Create a new untitled file or directory in path | ||||
path must be a directory | ||||
File extension can be specified. | ||||
Use `new` to create files with a fully specified path (including filename). | ||||
""" | ||||
path = path.strip('/') | ||||
if not self.dir_exists(path): | ||||
raise HTTPError(404, 'No such directory: %s' % path) | ||||
model = {} | ||||
if type: | ||||
model['type'] = type | ||||
if ext == '.ipynb': | ||||
model.setdefault('type', 'notebook') | ||||
else: | ||||
model.setdefault('type', 'file') | ||||
|
r18813 | insert = '' | ||
|
r18759 | if model['type'] == 'directory': | ||
untitled = self.untitled_directory | ||||
|
r18813 | insert = ' ' | ||
|
r18759 | elif model['type'] == 'notebook': | ||
untitled = self.untitled_notebook | ||||
ext = '.ipynb' | ||||
elif model['type'] == 'file': | ||||
untitled = self.untitled_file | ||||
else: | ||||
raise HTTPError(400, "Unexpected model type: %r" % model['type']) | ||||
|
r18813 | name = self.increment_filename(untitled + ext, path, insert=insert) | ||
|
r18759 | path = u'{0}/{1}'.format(path, name) | ||
return self.new(model, path) | ||||
def new(self, model=None, path=''): | ||||
|
r18758 | """Create a new file or directory and return its model with no content. | ||
|
r18759 | To create a new untitled entity in a directory, use `new_untitled`. | ||
|
r18758 | """ | ||
|
r13078 | path = path.strip('/') | ||
|
r13046 | if model is None: | ||
model = {} | ||||
|
r18758 | |||
|
r18759 | if path.endswith('.ipynb'): | ||
|
r18758 | model.setdefault('type', 'notebook') | ||
|
r18759 | else: | ||
model.setdefault('type', 'file') | ||||
|
r18758 | |||
# no content, not a directory, so fill out new-file model | ||||
if 'content' not in model and model['type'] != 'directory': | ||||
if model['type'] == 'notebook': | ||||
|
r18607 | model['content'] = new_notebook() | ||
|
r17532 | model['format'] = 'json' | ||
|
r17527 | else: | ||
model['content'] = '' | ||||
|
r17532 | model['type'] = 'file' | ||
model['format'] = 'text' | ||||
|
r18758 | |||
|
r18749 | model = self.save(model, path) | ||
|
r13046 | return model | ||
|
r8180 | |||
|
r18749 | def copy(self, from_path, to_path=None): | ||
|
r17524 | """Copy an existing file and return its new model. | ||
|
r17523 | |||
|
r18758 | If to_path not specified, it will be the parent directory of from_path. | ||
If to_path is a directory, filename will increment `from_path-Copy#.ext`. | ||||
|
r17535 | |||
|
r18758 | from_path must be a full path to a file. | ||
|
r13123 | """ | ||
|
r18749 | path = from_path.strip('/') | ||
|
r19606 | if to_path is not None: | ||
to_path = to_path.strip('/') | ||||
|
r18749 | if '/' in path: | ||
from_dir, from_name = path.rsplit('/', 1) | ||||
|
r17535 | else: | ||
|
r18749 | from_dir = '' | ||
from_name = path | ||||
|
r18791 | model = self.get(path) | ||
|
r18749 | model.pop('path', None) | ||
model.pop('name', None) | ||||
|
r17530 | if model['type'] == 'directory': | ||
raise HTTPError(400, "Can't copy directories") | ||||
|
r18749 | |||
|
r19607 | if to_path is None: | ||
|
r18749 | to_path = from_dir | ||
if self.dir_exists(to_path): | ||||
|
r18813 | name = copy_pat.sub(u'.', from_name) | ||
to_name = self.increment_filename(name, to_path, insert='-Copy') | ||||
|
r18754 | to_path = u'{0}/{1}'.format(to_path, to_name) | ||
|
r18749 | |||
model = self.save(model, to_path) | ||||
|
r13046 | return model | ||
|
r17523 | |||
|
r15293 | def log_info(self): | ||
self.log.info(self.info_string()) | ||||
|
r18749 | def trust_notebook(self, path): | ||
|
r15664 | """Explicitly trust a notebook | ||
|
r17523 | |||
|
r15664 | Parameters | ||
---------- | ||||
path : string | ||||
|
r18749 | The path of a notebook | ||
|
r15655 | """ | ||
|
r18791 | model = self.get(path) | ||
|
r15655 | nb = model['content'] | ||
|
r18749 | self.log.warn("Trusting notebook %s", path) | ||
|
r15655 | self.notary.mark_cells(nb, True) | ||
|
r18749 | self.save(model, path) | ||
|
r17523 | |||
|
r18749 | def check_and_sign(self, nb, path=''): | ||
|
r15293 | """Check for trusted cells, and sign the notebook. | ||
|
r17523 | |||
|
r15293 | Called as a part of saving notebooks. | ||
|
r17523 | |||
|
r15664 | Parameters | ||
---------- | ||||
nb : dict | ||||
|
r18612 | The notebook dict | ||
|
r15664 | path : string | ||
|
r18749 | The notebook's path (for logging) | ||
|
r10497 | """ | ||
|
r15293 | if self.notary.check_cells(nb): | ||
self.notary.sign(nb) | ||||
else: | ||||
|
r18749 | self.log.warn("Saving untrusted notebook %s", path) | ||
|
r17523 | |||
|
r18749 | def mark_trusted_cells(self, nb, path=''): | ||
|
r15293 | """Mark cells as trusted if the notebook signature matches. | ||
|
r17523 | |||
|
r15293 | Called as a part of loading notebooks. | ||
|
r17523 | |||
|
r15664 | Parameters | ||
---------- | ||||
nb : dict | ||||
|
r18607 | The notebook object (in current nbformat) | ||
|
r15664 | path : string | ||
|
r18758 | The notebook's path (for logging) | ||
|
r15293 | """ | ||
trusted = self.notary.check_signature(nb) | ||||
if not trusted: | ||||
|
r18749 | self.log.warn("Notebook %s is not trusted", path) | ||
|
r15293 | self.notary.mark_cells(nb, trusted) | ||
|
r8181 | |||
|
r15525 | def should_list(self, name): | ||
"""Should this file/directory name be displayed in a listing?""" | ||||
return not any(fnmatch(name, glob) for glob in self.hide_globs) | ||||
|
r19727 | |||
# Part 3: Checkpoints API | ||||
def create_checkpoint(self, path): | ||||
|
r19747 | """Create a checkpoint.""" | ||
|
r19839 | return self.checkpoints.create_checkpoint(self, path) | ||
|
r19727 | |||
def restore_checkpoint(self, checkpoint_id, path): | ||||
|
r19747 | """ | ||
Restore a checkpoint. | ||||
""" | ||||
|
r19839 | self.checkpoints.restore_checkpoint(self, checkpoint_id, path) | ||
|
r19828 | |||
def list_checkpoints(self, path): | ||||
|
r19839 | return self.checkpoints.list_checkpoints(path) | ||
|
r19747 | |||
|
r19727 | def delete_checkpoint(self, checkpoint_id, path): | ||
|
r19839 | return self.checkpoints.delete_checkpoint(checkpoint_id, path) | ||