##// END OF EJS Templates
Merge pull request #7324 from quantopian/separate-checkpoint-manager...
Min RK -
r19845:c3dd48be merge
parent child Browse files
Show More
@@ -0,0 +1,127 b''
1 """
2 Classes for managing Checkpoints.
3 """
4
5 # Copyright (c) IPython Development Team.
6 # Distributed under the terms of the Modified BSD License.
7
8 from tornado.web import HTTPError
9
10 from IPython.config.configurable import LoggingConfigurable
11
12
13 class Checkpoints(LoggingConfigurable):
14 """
15 Base class for managing checkpoints for a ContentsManager.
16
17 Subclasses are required to implement:
18
19 create_checkpoint(self, contents_mgr, path)
20 restore_checkpoint(self, contents_mgr, checkpoint_id, path)
21 rename_checkpoint(self, checkpoint_id, old_path, new_path)
22 delete_checkpoint(self, checkpoint_id, path)
23 list_checkpoints(self, path)
24 """
25 def create_checkpoint(self, contents_mgr, path):
26 """Create a checkpoint."""
27 raise NotImplementedError("must be implemented in a subclass")
28
29 def restore_checkpoint(self, contents_mgr, checkpoint_id, path):
30 """Restore a checkpoint"""
31 raise NotImplementedError("must be implemented in a subclass")
32
33 def rename_checkpoint(self, checkpoint_id, old_path, new_path):
34 """Rename a single checkpoint from old_path to new_path."""
35 raise NotImplementedError("must be implemented in a subclass")
36
37 def delete_checkpoint(self, checkpoint_id, path):
38 """delete a checkpoint for a file"""
39 raise NotImplementedError("must be implemented in a subclass")
40
41 def list_checkpoints(self, path):
42 """Return a list of checkpoints for a given file"""
43 raise NotImplementedError("must be implemented in a subclass")
44
45 def rename_all_checkpoints(self, old_path, new_path):
46 """Rename all checkpoints for old_path to new_path."""
47 for cp in self.list_checkpoints(old_path):
48 self.rename_checkpoint(cp['id'], old_path, new_path)
49
50 def delete_all_checkpoints(self, path):
51 """Delete all checkpoints for the given path."""
52 for checkpoint in self.list_checkpoints(path):
53 self.delete_checkpoint(checkpoint['id'], path)
54
55
56 class GenericCheckpointsMixin(object):
57 """
58 Helper for creating Checkpoints subclasses that can be used with any
59 ContentsManager.
60
61 Provides a ContentsManager-agnostic implementation of `create_checkpoint`
62 and `restore_checkpoint` in terms of the following operations:
63
64 - create_file_checkpoint(self, content, format, path)
65 - create_notebook_checkpoint(self, nb, path)
66 - get_file_checkpoint(self, checkpoint_id, path)
67 - get_notebook_checkpoint(self, checkpoint_id, path)
68
69 To create a generic CheckpointManager, add this mixin to a class that
70 implement the above three methods plus the remaining Checkpoints API
71 methods:
72
73 - delete_checkpoint(self, checkpoint_id, path)
74 - list_checkpoints(self, path)
75 - rename_checkpoint(self, checkpoint_id, old_path, new_path)
76 """
77
78 def create_checkpoint(self, contents_mgr, path):
79 model = contents_mgr.get(path, content=True)
80 type = model['type']
81 if type == 'notebook':
82 return self.create_notebook_checkpoint(
83 model['content'],
84 path,
85 )
86 elif type == 'file':
87 return self.create_file_checkpoint(
88 model['content'],
89 model['format'],
90 path,
91 )
92 else:
93 raise HTTPError(500, u'Unexpected type %s' % type)
94
95 def restore_checkpoint(self, contents_mgr, checkpoint_id, path):
96 """Restore a checkpoint."""
97 type = contents_mgr.get(path, content=False)['type']
98 if type == 'notebook':
99 model = self.get_notebook_checkpoint(checkpoint_id, path)
100 elif type == 'file':
101 model = self.get_file_checkpoint(checkpoint_id, path)
102 else:
103 raise HTTPError(500, u'Unexpected type %s' % type)
104 contents_mgr.save(model, path)
105
106 # Required Methods
107 def create_file_checkpoint(self, content, format, path):
108 """Create a checkpoint of the current state of a file
109
110 Returns a checkpoint model for the new checkpoint.
111 """
112 raise NotImplementedError("must be implemented in a subclass")
113
114 def create_notebook_checkpoint(self, nb, path):
115 """Create a checkpoint of the current state of a file
116
117 Returns a checkpoint model for the new checkpoint.
118 """
119 raise NotImplementedError("must be implemented in a subclass")
120
121 def get_checkpoint(self, checkpoint_id, path, type):
122 """Get the content of a checkpoint.
123
124 Returns an unvalidated model with the same structure as
125 the return value of ContentsManager.get
126 """
127 raise NotImplementedError("must be implemented in a subclass")
@@ -0,0 +1,200 b''
1 """
2 File-based Checkpoints implementations.
3 """
4 import os
5 import shutil
6
7 from tornado.web import HTTPError
8
9 from .checkpoints import (
10 Checkpoints,
11 GenericCheckpointsMixin,
12 )
13 from .fileio import FileManagerMixin
14
15 from IPython.utils import tz
16 from IPython.utils.path import ensure_dir_exists
17 from IPython.utils.py3compat import getcwd
18 from IPython.utils.traitlets import Unicode
19
20
21 class FileCheckpoints(FileManagerMixin, Checkpoints):
22 """
23 A Checkpoints that caches checkpoints for files in adjacent
24 directories.
25
26 Only works with FileContentsManager. Use GenericFileCheckpoints if
27 you want file-based checkpoints with another ContentsManager.
28 """
29
30 checkpoint_dir = Unicode(
31 '.ipynb_checkpoints',
32 config=True,
33 help="""The directory name in which to keep file checkpoints
34
35 This is a path relative to the file's own directory.
36
37 By default, it is .ipynb_checkpoints
38 """,
39 )
40
41 root_dir = Unicode(config=True)
42
43 def _root_dir_default(self):
44 try:
45 return self.parent.root_dir
46 except AttributeError:
47 return getcwd()
48
49 # ContentsManager-dependent checkpoint API
50 def create_checkpoint(self, contents_mgr, path):
51 """Create a checkpoint."""
52 checkpoint_id = u'checkpoint'
53 src_path = contents_mgr._get_os_path(path)
54 dest_path = self.checkpoint_path(checkpoint_id, path)
55 self._copy(src_path, dest_path)
56 return self.checkpoint_model(checkpoint_id, dest_path)
57
58 def restore_checkpoint(self, contents_mgr, checkpoint_id, path):
59 """Restore a checkpoint."""
60 src_path = self.checkpoint_path(checkpoint_id, path)
61 dest_path = contents_mgr._get_os_path(path)
62 self._copy(src_path, dest_path)
63
64 # ContentsManager-independent checkpoint API
65 def rename_checkpoint(self, checkpoint_id, old_path, new_path):
66 """Rename a checkpoint from old_path to new_path."""
67 old_cp_path = self.checkpoint_path(checkpoint_id, old_path)
68 new_cp_path = self.checkpoint_path(checkpoint_id, new_path)
69 if os.path.isfile(old_cp_path):
70 self.log.debug(
71 "Renaming checkpoint %s -> %s",
72 old_cp_path,
73 new_cp_path,
74 )
75 with self.perm_to_403():
76 shutil.move(old_cp_path, new_cp_path)
77
78 def delete_checkpoint(self, checkpoint_id, path):
79 """delete a file's checkpoint"""
80 path = path.strip('/')
81 cp_path = self.checkpoint_path(checkpoint_id, path)
82 if not os.path.isfile(cp_path):
83 self.no_such_checkpoint(path, checkpoint_id)
84
85 self.log.debug("unlinking %s", cp_path)
86 with self.perm_to_403():
87 os.unlink(cp_path)
88
89 def list_checkpoints(self, path):
90 """list the checkpoints for a given file
91
92 This contents manager currently only supports one checkpoint per file.
93 """
94 path = path.strip('/')
95 checkpoint_id = "checkpoint"
96 os_path = self.checkpoint_path(checkpoint_id, path)
97 if not os.path.isfile(os_path):
98 return []
99 else:
100 return [self.checkpoint_model(checkpoint_id, os_path)]
101
102 # Checkpoint-related utilities
103 def checkpoint_path(self, checkpoint_id, path):
104 """find the path to a checkpoint"""
105 path = path.strip('/')
106 parent, name = ('/' + path).rsplit('/', 1)
107 parent = parent.strip('/')
108 basename, ext = os.path.splitext(name)
109 filename = u"{name}-{checkpoint_id}{ext}".format(
110 name=basename,
111 checkpoint_id=checkpoint_id,
112 ext=ext,
113 )
114 os_path = self._get_os_path(path=parent)
115 cp_dir = os.path.join(os_path, self.checkpoint_dir)
116 with self.perm_to_403():
117 ensure_dir_exists(cp_dir)
118 cp_path = os.path.join(cp_dir, filename)
119 return cp_path
120
121 def checkpoint_model(self, checkpoint_id, os_path):
122 """construct the info dict for a given checkpoint"""
123 stats = os.stat(os_path)
124 last_modified = tz.utcfromtimestamp(stats.st_mtime)
125 info = dict(
126 id=checkpoint_id,
127 last_modified=last_modified,
128 )
129 return info
130
131 # Error Handling
132 def no_such_checkpoint(self, path, checkpoint_id):
133 raise HTTPError(
134 404,
135 u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id)
136 )
137
138
139 class GenericFileCheckpoints(GenericCheckpointsMixin, FileCheckpoints):
140 """
141 Local filesystem Checkpoints that works with any conforming
142 ContentsManager.
143 """
144 def create_file_checkpoint(self, content, format, path):
145 """Create a checkpoint from the current content of a notebook."""
146 path = path.strip('/')
147 # only the one checkpoint ID:
148 checkpoint_id = u"checkpoint"
149 os_checkpoint_path = self.checkpoint_path(checkpoint_id, path)
150 self.log.debug("creating checkpoint for %s", path)
151 with self.perm_to_403():
152 self._save_file(os_checkpoint_path, content, format=format)
153
154 # return the checkpoint info
155 return self.checkpoint_model(checkpoint_id, os_checkpoint_path)
156
157 def create_notebook_checkpoint(self, nb, path):
158 """Create a checkpoint from the current content of a notebook."""
159 path = path.strip('/')
160 # only the one checkpoint ID:
161 checkpoint_id = u"checkpoint"
162 os_checkpoint_path = self.checkpoint_path(checkpoint_id, path)
163 self.log.debug("creating checkpoint for %s", path)
164 with self.perm_to_403():
165 self._save_notebook(os_checkpoint_path, nb)
166
167 # return the checkpoint info
168 return self.checkpoint_model(checkpoint_id, os_checkpoint_path)
169
170 def get_notebook_checkpoint(self, checkpoint_id, path):
171
172 path = path.strip('/')
173 self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
174 os_checkpoint_path = self.checkpoint_path(checkpoint_id, path)
175
176 if not os.path.isfile(os_checkpoint_path):
177 self.no_such_checkpoint(path, checkpoint_id)
178
179 return {
180 'type': 'notebook',
181 'content': self._read_notebook(
182 os_checkpoint_path,
183 as_version=4,
184 ),
185 }
186
187 def get_file_checkpoint(self, checkpoint_id, path):
188 path = path.strip('/')
189 self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
190 os_checkpoint_path = self.checkpoint_path(checkpoint_id, path)
191
192 if not os.path.isfile(os_checkpoint_path):
193 self.no_such_checkpoint(path, checkpoint_id)
194
195 content, format = self._read_file(os_checkpoint_path, format=None)
196 return {
197 'type': 'file',
198 'content': content,
199 'format': format,
200 }
@@ -0,0 +1,166 b''
1 """
2 Utilities for file-based Contents/Checkpoints managers.
3 """
4
5 # Copyright (c) IPython Development Team.
6 # Distributed under the terms of the Modified BSD License.
7
8 import base64
9 from contextlib import contextmanager
10 import errno
11 import io
12 import os
13 import shutil
14
15 from tornado.web import HTTPError
16
17 from IPython.html.utils import (
18 to_api_path,
19 to_os_path,
20 )
21 from IPython import nbformat
22 from IPython.utils.io import atomic_writing
23 from IPython.utils.py3compat import str_to_unicode
24
25
26 class FileManagerMixin(object):
27 """
28 Mixin for ContentsAPI classes that interact with the filesystem.
29
30 Provides facilities for reading, writing, and copying both notebooks and
31 generic files.
32
33 Shared by FileContentsManager and FileCheckpoints.
34
35 Note
36 ----
37 Classes using this mixin must provide the following attributes:
38
39 root_dir : unicode
40 A directory against against which API-style paths are to be resolved.
41
42 log : logging.Logger
43 """
44
45 @contextmanager
46 def open(self, os_path, *args, **kwargs):
47 """wrapper around io.open that turns permission errors into 403"""
48 with self.perm_to_403(os_path):
49 with io.open(os_path, *args, **kwargs) as f:
50 yield f
51
52 @contextmanager
53 def atomic_writing(self, os_path, *args, **kwargs):
54 """wrapper around atomic_writing that turns permission errors to 403"""
55 with self.perm_to_403(os_path):
56 with atomic_writing(os_path, *args, **kwargs) as f:
57 yield f
58
59 @contextmanager
60 def perm_to_403(self, os_path=''):
61 """context manager for turning permission errors into 403."""
62 try:
63 yield
64 except OSError as e:
65 if e.errno in {errno.EPERM, errno.EACCES}:
66 # make 403 error message without root prefix
67 # this may not work perfectly on unicode paths on Python 2,
68 # but nobody should be doing that anyway.
69 if not os_path:
70 os_path = str_to_unicode(e.filename or 'unknown file')
71 path = to_api_path(os_path, root=self.root_dir)
72 raise HTTPError(403, u'Permission denied: %s' % path)
73 else:
74 raise
75
76 def _copy(self, src, dest):
77 """copy src to dest
78
79 like shutil.copy2, but log errors in copystat
80 """
81 shutil.copyfile(src, dest)
82 try:
83 shutil.copystat(src, dest)
84 except OSError:
85 self.log.debug("copystat on %s failed", dest, exc_info=True)
86
87 def _get_os_path(self, path):
88 """Given an API path, return its file system path.
89
90 Parameters
91 ----------
92 path : string
93 The relative API path to the named file.
94
95 Returns
96 -------
97 path : string
98 Native, absolute OS path to for a file.
99 """
100 return to_os_path(path, self.root_dir)
101
102 def _read_notebook(self, os_path, as_version=4):
103 """Read a notebook from an os path."""
104 with self.open(os_path, 'r', encoding='utf-8') as f:
105 try:
106 return nbformat.read(f, as_version=as_version)
107 except Exception as e:
108 raise HTTPError(
109 400,
110 u"Unreadable Notebook: %s %r" % (os_path, e),
111 )
112
113 def _save_notebook(self, os_path, nb):
114 """Save a notebook to an os_path."""
115 with self.atomic_writing(os_path, encoding='utf-8') as f:
116 nbformat.write(nb, f, version=nbformat.NO_CONVERT)
117
118 def _read_file(self, os_path, format):
119 """Read a non-notebook file.
120
121 os_path: The path to be read.
122 format:
123 If 'text', the contents will be decoded as UTF-8.
124 If 'base64', the raw bytes contents will be encoded as base64.
125 If not specified, try to decode as UTF-8, and fall back to base64
126 """
127 if not os.path.isfile(os_path):
128 raise HTTPError(400, "Cannot read non-file %s" % os_path)
129
130 with self.open(os_path, 'rb') as f:
131 bcontent = f.read()
132
133 if format is None or format == 'text':
134 # Try to interpret as unicode if format is unknown or if unicode
135 # was explicitly requested.
136 try:
137 return bcontent.decode('utf8'), 'text'
138 except UnicodeError:
139 if format == 'text':
140 raise HTTPError(
141 400,
142 "%s is not UTF-8 encoded" % os_path,
143 reason='bad format',
144 )
145 return base64.encodestring(bcontent).decode('ascii'), 'base64'
146
147 def _save_file(self, os_path, content, format):
148 """Save content of a generic file."""
149 if format not in {'text', 'base64'}:
150 raise HTTPError(
151 400,
152 "Must specify format of file contents as 'text' or 'base64'",
153 )
154 try:
155 if format == 'text':
156 bcontent = content.encode('utf8')
157 else:
158 b64_bytes = content.encode('ascii')
159 bcontent = base64.decodestring(b64_bytes)
160 except Exception as e:
161 raise HTTPError(
162 400, u'Encoding error saving %s: %s' % (os_path, e)
163 )
164
165 with self.atomic_writing(os_path, text=False) as f:
166 f.write(bcontent)
@@ -3,28 +3,31 b''
3 # Copyright (c) IPython Development Team.
3 # Copyright (c) IPython Development Team.
4 # Distributed under the terms of the Modified BSD License.
4 # Distributed under the terms of the Modified BSD License.
5
5
6 import base64
6
7 import errno
8 import io
7 import io
9 import os
8 import os
10 import shutil
9 import shutil
11 from contextlib import contextmanager
12 import mimetypes
10 import mimetypes
13
11
14 from tornado import web
12 from tornado import web
15
13
14 from .filecheckpoints import FileCheckpoints
15 from .fileio import FileManagerMixin
16 from .manager import ContentsManager
16 from .manager import ContentsManager
17
17 from IPython import nbformat
18 from IPython import nbformat
18 from IPython.utils.io import atomic_writing
19 from IPython.utils.importstring import import_item
19 from IPython.utils.importstring import import_item
20 from IPython.utils.path import ensure_dir_exists
21 from IPython.utils.traitlets import Any, Unicode, Bool, TraitError
20 from IPython.utils.traitlets import Any, Unicode, Bool, TraitError
22 from IPython.utils.py3compat import getcwd, str_to_unicode, string_types
21 from IPython.utils.py3compat import getcwd, string_types
23 from IPython.utils import tz
22 from IPython.utils import tz
24 from IPython.html.utils import is_hidden, to_os_path, to_api_path
23 from IPython.html.utils import (
24 is_hidden,
25 to_api_path,
26 )
25
27
26 _script_exporter = None
28 _script_exporter = None
27
29
30
28 def _post_save_script(model, os_path, contents_manager, **kwargs):
31 def _post_save_script(model, os_path, contents_manager, **kwargs):
29 """convert notebooks to Python script after save with nbconvert
32 """convert notebooks to Python script after save with nbconvert
30
33
@@ -48,7 +51,8 b' def _post_save_script(model, os_path, contents_manager, **kwargs):'
48 with io.open(script_fname, 'w', encoding='utf-8') as f:
51 with io.open(script_fname, 'w', encoding='utf-8') as f:
49 f.write(script)
52 f.write(script)
50
53
51 class FileContentsManager(ContentsManager):
54
55 class FileContentsManager(FileManagerMixin, ContentsManager):
52
56
53 root_dir = Unicode(config=True)
57 root_dir = Unicode(config=True)
54
58
@@ -57,38 +61,7 b' class FileContentsManager(ContentsManager):'
57 return self.parent.notebook_dir
61 return self.parent.notebook_dir
58 except AttributeError:
62 except AttributeError:
59 return getcwd()
63 return getcwd()
60
64
61 @contextmanager
62 def perm_to_403(self, os_path=''):
63 """context manager for turning permission errors into 403"""
64 try:
65 yield
66 except OSError as e:
67 if e.errno in {errno.EPERM, errno.EACCES}:
68 # make 403 error message without root prefix
69 # this may not work perfectly on unicode paths on Python 2,
70 # but nobody should be doing that anyway.
71 if not os_path:
72 os_path = str_to_unicode(e.filename or 'unknown file')
73 path = to_api_path(os_path, self.root_dir)
74 raise web.HTTPError(403, u'Permission denied: %s' % path)
75 else:
76 raise
77
78 @contextmanager
79 def open(self, os_path, *args, **kwargs):
80 """wrapper around io.open that turns permission errors into 403"""
81 with self.perm_to_403(os_path):
82 with io.open(os_path, *args, **kwargs) as f:
83 yield f
84
85 @contextmanager
86 def atomic_writing(self, os_path, *args, **kwargs):
87 """wrapper around atomic_writing that turns permission errors into 403"""
88 with self.perm_to_403(os_path):
89 with atomic_writing(os_path, *args, **kwargs) as f:
90 yield f
91
92 save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook')
65 save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook')
93 def _save_script_changed(self):
66 def _save_script_changed(self):
94 self.log.warn("""
67 self.log.warn("""
@@ -148,60 +121,8 b' class FileContentsManager(ContentsManager):'
148 if not os.path.isdir(new):
121 if not os.path.isdir(new):
149 raise TraitError("%r is not a directory" % new)
122 raise TraitError("%r is not a directory" % new)
150
123
151 checkpoint_dir = Unicode('.ipynb_checkpoints', config=True,
124 def _checkpoints_class_default(self):
152 help="""The directory name in which to keep file checkpoints
125 return FileCheckpoints
153
154 This is a path relative to the file's own directory.
155
156 By default, it is .ipynb_checkpoints
157 """
158 )
159
160 def _copy(self, src, dest):
161 """copy src to dest
162
163 like shutil.copy2, but log errors in copystat
164 """
165 shutil.copyfile(src, dest)
166 try:
167 shutil.copystat(src, dest)
168 except OSError as e:
169 self.log.debug("copystat on %s failed", dest, exc_info=True)
170
171 def _get_os_path(self, path):
172 """Given an API path, return its file system path.
173
174 Parameters
175 ----------
176 path : string
177 The relative API path to the named file.
178
179 Returns
180 -------
181 path : string
182 Native, absolute OS path to for a file.
183 """
184 return to_os_path(path, self.root_dir)
185
186 def dir_exists(self, path):
187 """Does the API-style path refer to an extant directory?
188
189 API-style wrapper for os.path.isdir
190
191 Parameters
192 ----------
193 path : string
194 The path to check. This is an API path (`/` separated,
195 relative to root_dir).
196
197 Returns
198 -------
199 exists : bool
200 Whether the path is indeed a directory.
201 """
202 path = path.strip('/')
203 os_path = self._get_os_path(path=path)
204 return os.path.isdir(os_path)
205
126
206 def is_hidden(self, path):
127 def is_hidden(self, path):
207 """Does the API style path correspond to a hidden directory or file?
128 """Does the API style path correspond to a hidden directory or file?
@@ -240,6 +161,26 b' class FileContentsManager(ContentsManager):'
240 os_path = self._get_os_path(path)
161 os_path = self._get_os_path(path)
241 return os.path.isfile(os_path)
162 return os.path.isfile(os_path)
242
163
164 def dir_exists(self, path):
165 """Does the API-style path refer to an extant directory?
166
167 API-style wrapper for os.path.isdir
168
169 Parameters
170 ----------
171 path : string
172 The path to check. This is an API path (`/` separated,
173 relative to root_dir).
174
175 Returns
176 -------
177 exists : bool
178 Whether the path is indeed a directory.
179 """
180 path = path.strip('/')
181 os_path = self._get_os_path(path=path)
182 return os.path.isdir(os_path)
183
243 def exists(self, path):
184 def exists(self, path):
244 """Returns True if the path exists, else returns False.
185 """Returns True if the path exists, else returns False.
245
186
@@ -338,33 +279,20 b' class FileContentsManager(ContentsManager):'
338 os_path = self._get_os_path(path)
279 os_path = self._get_os_path(path)
339
280
340 if content:
281 if content:
341 if not os.path.isfile(os_path):
282 content, format = self._read_file(os_path, format)
342 # could be FIFO
283 default_mime = {
343 raise web.HTTPError(400, "Cannot get content of non-file %s" % os_path)
284 'text': 'text/plain',
344 with self.open(os_path, 'rb') as f:
285 'base64': 'application/octet-stream'
345 bcontent = f.read()
286 }[format]
346
287
347 if format != 'base64':
288 model.update(
348 try:
289 content=content,
349 model['content'] = bcontent.decode('utf8')
290 format=format,
350 except UnicodeError as e:
291 mimetype=mimetypes.guess_type(os_path)[0] or default_mime,
351 if format == 'text':
292 )
352 raise web.HTTPError(400, "%s is not UTF-8 encoded" % path, reason='bad format')
353 else:
354 model['format'] = 'text'
355 default_mime = 'text/plain'
356
357 if model['content'] is None:
358 model['content'] = base64.encodestring(bcontent).decode('ascii')
359 model['format'] = 'base64'
360 if model['format'] == 'base64':
361 default_mime = 'application/octet-stream'
362
363 model['mimetype'] = mimetypes.guess_type(os_path)[0] or default_mime
364
293
365 return model
294 return model
366
295
367
368 def _notebook_model(self, path, content=True):
296 def _notebook_model(self, path, content=True):
369 """Build a notebook model
297 """Build a notebook model
370
298
@@ -375,11 +303,7 b' class FileContentsManager(ContentsManager):'
375 model['type'] = 'notebook'
303 model['type'] = 'notebook'
376 if content:
304 if content:
377 os_path = self._get_os_path(path)
305 os_path = self._get_os_path(path)
378 with self.open(os_path, 'r', encoding='utf-8') as f:
306 nb = self._read_notebook(os_path, as_version=4)
379 try:
380 nb = nbformat.read(f, as_version=4)
381 except Exception as e:
382 raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e))
383 self.mark_trusted_cells(nb, path)
307 self.mark_trusted_cells(nb, path)
384 model['content'] = nb
308 model['content'] = nb
385 model['format'] = 'json'
309 model['format'] = 'json'
@@ -428,33 +352,6 b' class FileContentsManager(ContentsManager):'
428 model = self._file_model(path, content=content, format=format)
352 model = self._file_model(path, content=content, format=format)
429 return model
353 return model
430
354
431 def _save_notebook(self, os_path, model, path=''):
432 """save a notebook file"""
433 # Save the notebook file
434 nb = nbformat.from_dict(model['content'])
435
436 self.check_and_sign(nb, path)
437
438 with self.atomic_writing(os_path, encoding='utf-8') as f:
439 nbformat.write(nb, f, version=nbformat.NO_CONVERT)
440
441 def _save_file(self, os_path, model, path=''):
442 """save a non-notebook file"""
443 fmt = model.get('format', None)
444 if fmt not in {'text', 'base64'}:
445 raise web.HTTPError(400, "Must specify format of file contents as 'text' or 'base64'")
446 try:
447 content = model['content']
448 if fmt == 'text':
449 bcontent = content.encode('utf8')
450 else:
451 b64_bytes = content.encode('ascii')
452 bcontent = base64.decodestring(b64_bytes)
453 except Exception as e:
454 raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e))
455 with self.atomic_writing(os_path, text=False) as f:
456 f.write(bcontent)
457
458 def _save_directory(self, os_path, model, path=''):
355 def _save_directory(self, os_path, model, path=''):
459 """create a directory"""
356 """create a directory"""
460 if is_hidden(os_path, self.root_dir):
357 if is_hidden(os_path, self.root_dir):
@@ -478,17 +375,19 b' class FileContentsManager(ContentsManager):'
478
375
479 self.run_pre_save_hook(model=model, path=path)
376 self.run_pre_save_hook(model=model, path=path)
480
377
481 # One checkpoint should always exist
482 if self.file_exists(path) and not self.list_checkpoints(path):
483 self.create_checkpoint(path)
484
485 os_path = self._get_os_path(path)
378 os_path = self._get_os_path(path)
486 self.log.debug("Saving %s", os_path)
379 self.log.debug("Saving %s", os_path)
487 try:
380 try:
488 if model['type'] == 'notebook':
381 if model['type'] == 'notebook':
489 self._save_notebook(os_path, model, path)
382 nb = nbformat.from_dict(model['content'])
383 self.check_and_sign(nb, path)
384 self._save_notebook(os_path, nb)
385 # One checkpoint should always exist for notebooks.
386 if not self.checkpoints.list_checkpoints(path):
387 self.create_checkpoint(path)
490 elif model['type'] == 'file':
388 elif model['type'] == 'file':
491 self._save_file(os_path, model, path)
389 # Missing format will be handled internally by _save_file.
390 self._save_file(os_path, model['content'], model.get('format'))
492 elif model['type'] == 'directory':
391 elif model['type'] == 'directory':
493 self._save_directory(os_path, model, path)
392 self._save_directory(os_path, model, path)
494 else:
393 else:
@@ -512,28 +411,23 b' class FileContentsManager(ContentsManager):'
512
411
513 return model
412 return model
514
413
515 def delete(self, path):
414 def delete_file(self, path):
516 """Delete file at path."""
415 """Delete file at path."""
517 path = path.strip('/')
416 path = path.strip('/')
518 os_path = self._get_os_path(path)
417 os_path = self._get_os_path(path)
519 rm = os.unlink
418 rm = os.unlink
520 if os.path.isdir(os_path):
419 if os.path.isdir(os_path):
521 listing = os.listdir(os_path)
420 listing = os.listdir(os_path)
522 # don't delete non-empty directories (checkpoints dir doesn't count)
421 # Don't delete non-empty directories.
523 if listing and listing != [self.checkpoint_dir]:
422 # A directory containing only leftover checkpoints is
524 raise web.HTTPError(400, u'Directory %s not empty' % os_path)
423 # considered empty.
424 cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None)
425 for entry in listing:
426 if entry != cp_dir:
427 raise web.HTTPError(400, u'Directory %s not empty' % os_path)
525 elif not os.path.isfile(os_path):
428 elif not os.path.isfile(os_path):
526 raise web.HTTPError(404, u'File does not exist: %s' % os_path)
429 raise web.HTTPError(404, u'File does not exist: %s' % os_path)
527
430
528 # clear checkpoints
529 for checkpoint in self.list_checkpoints(path):
530 checkpoint_id = checkpoint['id']
531 cp_path = self.get_checkpoint_path(checkpoint_id, path)
532 if os.path.isfile(cp_path):
533 self.log.debug("Unlinking checkpoint %s", cp_path)
534 with self.perm_to_403():
535 rm(cp_path)
536
537 if os.path.isdir(os_path):
431 if os.path.isdir(os_path):
538 self.log.debug("Removing directory %s", os_path)
432 self.log.debug("Removing directory %s", os_path)
539 with self.perm_to_403():
433 with self.perm_to_403():
@@ -543,7 +437,7 b' class FileContentsManager(ContentsManager):'
543 with self.perm_to_403():
437 with self.perm_to_403():
544 rm(os_path)
438 rm(os_path)
545
439
546 def rename(self, old_path, new_path):
440 def rename_file(self, old_path, new_path):
547 """Rename a file."""
441 """Rename a file."""
548 old_path = old_path.strip('/')
442 old_path = old_path.strip('/')
549 new_path = new_path.strip('/')
443 new_path = new_path.strip('/')
@@ -566,111 +460,6 b' class FileContentsManager(ContentsManager):'
566 except Exception as e:
460 except Exception as e:
567 raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e))
461 raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e))
568
462
569 # Move the checkpoints
570 old_checkpoints = self.list_checkpoints(old_path)
571 for cp in old_checkpoints:
572 checkpoint_id = cp['id']
573 old_cp_path = self.get_checkpoint_path(checkpoint_id, old_path)
574 new_cp_path = self.get_checkpoint_path(checkpoint_id, new_path)
575 if os.path.isfile(old_cp_path):
576 self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path)
577 with self.perm_to_403():
578 shutil.move(old_cp_path, new_cp_path)
579
580 # Checkpoint-related utilities
581
582 def get_checkpoint_path(self, checkpoint_id, path):
583 """find the path to a checkpoint"""
584 path = path.strip('/')
585 parent, name = ('/' + path).rsplit('/', 1)
586 parent = parent.strip('/')
587 basename, ext = os.path.splitext(name)
588 filename = u"{name}-{checkpoint_id}{ext}".format(
589 name=basename,
590 checkpoint_id=checkpoint_id,
591 ext=ext,
592 )
593 os_path = self._get_os_path(path=parent)
594 cp_dir = os.path.join(os_path, self.checkpoint_dir)
595 with self.perm_to_403():
596 ensure_dir_exists(cp_dir)
597 cp_path = os.path.join(cp_dir, filename)
598 return cp_path
599
600 def get_checkpoint_model(self, checkpoint_id, path):
601 """construct the info dict for a given checkpoint"""
602 path = path.strip('/')
603 cp_path = self.get_checkpoint_path(checkpoint_id, path)
604 stats = os.stat(cp_path)
605 last_modified = tz.utcfromtimestamp(stats.st_mtime)
606 info = dict(
607 id = checkpoint_id,
608 last_modified = last_modified,
609 )
610 return info
611
612 # public checkpoint API
613
614 def create_checkpoint(self, path):
615 """Create a checkpoint from the current state of a file"""
616 path = path.strip('/')
617 if not self.file_exists(path):
618 raise web.HTTPError(404)
619 src_path = self._get_os_path(path)
620 # only the one checkpoint ID:
621 checkpoint_id = u"checkpoint"
622 cp_path = self.get_checkpoint_path(checkpoint_id, path)
623 self.log.debug("creating checkpoint for %s", path)
624 with self.perm_to_403():
625 self._copy(src_path, cp_path)
626
627 # return the checkpoint info
628 return self.get_checkpoint_model(checkpoint_id, path)
629
630 def list_checkpoints(self, path):
631 """list the checkpoints for a given file
632
633 This contents manager currently only supports one checkpoint per file.
634 """
635 path = path.strip('/')
636 checkpoint_id = "checkpoint"
637 os_path = self.get_checkpoint_path(checkpoint_id, path)
638 if not os.path.exists(os_path):
639 return []
640 else:
641 return [self.get_checkpoint_model(checkpoint_id, path)]
642
643
644 def restore_checkpoint(self, checkpoint_id, path):
645 """restore a file to a checkpointed state"""
646 path = path.strip('/')
647 self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
648 nb_path = self._get_os_path(path)
649 cp_path = self.get_checkpoint_path(checkpoint_id, path)
650 if not os.path.isfile(cp_path):
651 self.log.debug("checkpoint file does not exist: %s", cp_path)
652 raise web.HTTPError(404,
653 u'checkpoint does not exist: %s@%s' % (path, checkpoint_id)
654 )
655 # ensure notebook is readable (never restore from an unreadable notebook)
656 if cp_path.endswith('.ipynb'):
657 with self.open(cp_path, 'r', encoding='utf-8') as f:
658 nbformat.read(f, as_version=4)
659 self.log.debug("copying %s -> %s", cp_path, nb_path)
660 with self.perm_to_403():
661 self._copy(cp_path, nb_path)
662
663 def delete_checkpoint(self, checkpoint_id, path):
664 """delete a file's checkpoint"""
665 path = path.strip('/')
666 cp_path = self.get_checkpoint_path(checkpoint_id, path)
667 if not os.path.isfile(cp_path):
668 raise web.HTTPError(404,
669 u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id)
670 )
671 self.log.debug("unlinking %s", cp_path)
672 os.unlink(cp_path)
673
674 def info_string(self):
463 def info_string(self):
675 return "Serving notebooks from local directory: %s" % self.root_dir
464 return "Serving notebooks from local directory: %s" % self.root_dir
676
465
@@ -11,15 +11,25 b' import re'
11
11
12 from tornado.web import HTTPError
12 from tornado.web import HTTPError
13
13
14 from .checkpoints import Checkpoints
14 from IPython.config.configurable import LoggingConfigurable
15 from IPython.config.configurable import LoggingConfigurable
15 from IPython.nbformat import sign, validate, ValidationError
16 from IPython.nbformat import sign, validate, ValidationError
16 from IPython.nbformat.v4 import new_notebook
17 from IPython.nbformat.v4 import new_notebook
17 from IPython.utils.importstring import import_item
18 from IPython.utils.importstring import import_item
18 from IPython.utils.traitlets import Instance, Unicode, List, Any, TraitError
19 from IPython.utils.traitlets import (
20 Any,
21 Dict,
22 Instance,
23 List,
24 TraitError,
25 Type,
26 Unicode,
27 )
19 from IPython.utils.py3compat import string_types
28 from IPython.utils.py3compat import string_types
20
29
21 copy_pat = re.compile(r'\-Copy\d*\.')
30 copy_pat = re.compile(r'\-Copy\d*\.')
22
31
32
23 class ContentsManager(LoggingConfigurable):
33 class ContentsManager(LoggingConfigurable):
24 """Base class for serving files and directories.
34 """Base class for serving files and directories.
25
35
@@ -97,6 +107,19 b' class ContentsManager(LoggingConfigurable):'
97 except Exception:
107 except Exception:
98 self.log.error("Pre-save hook failed on %s", path, exc_info=True)
108 self.log.error("Pre-save hook failed on %s", path, exc_info=True)
99
109
110 checkpoints_class = Type(Checkpoints, config=True)
111 checkpoints = Instance(Checkpoints, config=True)
112 checkpoints_kwargs = Dict(allow_none=False, config=True)
113
114 def _checkpoints_default(self):
115 return self.checkpoints_class(**self.checkpoints_kwargs)
116
117 def _checkpoints_kwargs_default(self):
118 return dict(
119 parent=self,
120 log=self.log,
121 )
122
100 # ContentsManager API part 1: methods that must be
123 # ContentsManager API part 1: methods that must be
101 # implemented in subclasses.
124 # implemented in subclasses.
102
125
@@ -186,32 +209,27 b' class ContentsManager(LoggingConfigurable):'
186 """
209 """
187 raise NotImplementedError('must be implemented in a subclass')
210 raise NotImplementedError('must be implemented in a subclass')
188
211
189 def delete(self, path):
212 def delete_file(self, path):
190 """Delete file or directory by path."""
213 """Delete file or directory by path."""
191 raise NotImplementedError('must be implemented in a subclass')
214 raise NotImplementedError('must be implemented in a subclass')
192
215
193 def create_checkpoint(self, path):
216 def rename_file(self, old_path, new_path):
194 """Create a checkpoint of the current state of a file
217 """Rename a file."""
195
218 raise NotImplementedError('must be implemented in a subclass')
196 Returns a checkpoint_id for the new checkpoint.
197 """
198 raise NotImplementedError("must be implemented in a subclass")
199
200 def list_checkpoints(self, path):
201 """Return a list of checkpoints for a given file"""
202 return []
203
204 def restore_checkpoint(self, checkpoint_id, path):
205 """Restore a file from one of its checkpoints"""
206 raise NotImplementedError("must be implemented in a subclass")
207
208 def delete_checkpoint(self, checkpoint_id, path):
209 """delete a checkpoint for a file"""
210 raise NotImplementedError("must be implemented in a subclass")
211
219
212 # ContentsManager API part 2: methods that have useable default
220 # ContentsManager API part 2: methods that have useable default
213 # implementations, but can be overridden in subclasses.
221 # implementations, but can be overridden in subclasses.
214
222
223 def delete(self, path):
224 """Delete a file/directory and any associated checkpoints."""
225 self.delete_file(path)
226 self.checkpoints.delete_all_checkpoints(path)
227
228 def rename(self, old_path, new_path):
229 """Rename a file and any checkpoints associated with that file."""
230 self.rename_file(old_path, new_path)
231 self.checkpoints.rename_all_checkpoints(old_path, new_path)
232
215 def update(self, model, path):
233 def update(self, model, path):
216 """Update the file's path
234 """Update the file's path
217
235
@@ -431,3 +449,20 b' class ContentsManager(LoggingConfigurable):'
431 def should_list(self, name):
449 def should_list(self, name):
432 """Should this file/directory name be displayed in a listing?"""
450 """Should this file/directory name be displayed in a listing?"""
433 return not any(fnmatch(name, glob) for glob in self.hide_globs)
451 return not any(fnmatch(name, glob) for glob in self.hide_globs)
452
453 # Part 3: Checkpoints API
454 def create_checkpoint(self, path):
455 """Create a checkpoint."""
456 return self.checkpoints.create_checkpoint(self, path)
457
458 def restore_checkpoint(self, checkpoint_id, path):
459 """
460 Restore a checkpoint.
461 """
462 self.checkpoints.restore_checkpoint(self, checkpoint_id, path)
463
464 def list_checkpoints(self, path):
465 return self.checkpoints.list_checkpoints(path)
466
467 def delete_checkpoint(self, checkpoint_id, path):
468 return self.checkpoints.delete_checkpoint(checkpoint_id, path)
@@ -2,6 +2,7 b''
2 """Test the contents webservice API."""
2 """Test the contents webservice API."""
3
3
4 import base64
4 import base64
5 from contextlib import contextmanager
5 import io
6 import io
6 import json
7 import json
7 import os
8 import os
@@ -12,6 +13,9 b' pjoin = os.path.join'
12
13
13 import requests
14 import requests
14
15
16 from ..filecheckpoints import GenericFileCheckpoints
17
18 from IPython.config import Config
15 from IPython.html.utils import url_path_join, url_escape, to_os_path
19 from IPython.html.utils import url_path_join, url_escape, to_os_path
16 from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error
20 from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error
17 from IPython.nbformat import read, write, from_dict
21 from IPython.nbformat import read, write, from_dict
@@ -21,6 +25,7 b' from IPython.nbformat.v4 import ('
21 from IPython.nbformat import v2
25 from IPython.nbformat import v2
22 from IPython.utils import py3compat
26 from IPython.utils import py3compat
23 from IPython.utils.data import uniq_stable
27 from IPython.utils.data import uniq_stable
28 from IPython.utils.tempdir import TemporaryDirectory
24
29
25
30
26 def notebooks_only(dir_model):
31 def notebooks_only(dir_model):
@@ -502,7 +507,6 b' class APITest(NotebookTestBase):'
502 self.assertEqual(newnb.cells[0].source,
507 self.assertEqual(newnb.cells[0].source,
503 u'Created by test ³')
508 u'Created by test ³')
504
509
505
506 def test_checkpoints(self):
510 def test_checkpoints(self):
507 resp = self.api.read('foo/a.ipynb')
511 resp = self.api.read('foo/a.ipynb')
508 r = self.api.new_checkpoint('foo/a.ipynb')
512 r = self.api.new_checkpoint('foo/a.ipynb')
@@ -540,3 +544,93 b' class APITest(NotebookTestBase):'
540 self.assertEqual(r.status_code, 204)
544 self.assertEqual(r.status_code, 204)
541 cps = self.api.get_checkpoints('foo/a.ipynb').json()
545 cps = self.api.get_checkpoints('foo/a.ipynb').json()
542 self.assertEqual(cps, [])
546 self.assertEqual(cps, [])
547
548 def test_file_checkpoints(self):
549 """
550 Test checkpointing of non-notebook files.
551 """
552 filename = 'foo/a.txt'
553 resp = self.api.read(filename)
554 orig_content = json.loads(resp.text)['content']
555
556 # Create a checkpoint.
557 r = self.api.new_checkpoint(filename)
558 self.assertEqual(r.status_code, 201)
559 cp1 = r.json()
560 self.assertEqual(set(cp1), {'id', 'last_modified'})
561 self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id'])
562
563 # Modify the file and save.
564 new_content = orig_content + '\nsecond line'
565 model = {
566 'content': new_content,
567 'type': 'file',
568 'format': 'text',
569 }
570 resp = self.api.save(filename, body=json.dumps(model))
571
572 # List checkpoints
573 cps = self.api.get_checkpoints(filename).json()
574 self.assertEqual(cps, [cp1])
575
576 content = self.api.read(filename).json()['content']
577 self.assertEqual(content, new_content)
578
579 # Restore cp1
580 r = self.api.restore_checkpoint(filename, cp1['id'])
581 self.assertEqual(r.status_code, 204)
582 restored_content = self.api.read(filename).json()['content']
583 self.assertEqual(restored_content, orig_content)
584
585 # Delete cp1
586 r = self.api.delete_checkpoint(filename, cp1['id'])
587 self.assertEqual(r.status_code, 204)
588 cps = self.api.get_checkpoints(filename).json()
589 self.assertEqual(cps, [])
590
591 @contextmanager
592 def patch_cp_root(self, dirname):
593 """
594 Temporarily patch the root dir of our checkpoint manager.
595 """
596 cpm = self.notebook.contents_manager.checkpoints
597 old_dirname = cpm.root_dir
598 cpm.root_dir = dirname
599 try:
600 yield
601 finally:
602 cpm.root_dir = old_dirname
603
604 def test_checkpoints_separate_root(self):
605 """
606 Test that FileCheckpoints functions correctly even when it's
607 using a different root dir from FileContentsManager. This also keeps
608 the implementation honest for use with ContentsManagers that don't map
609 models to the filesystem
610
611 Override this method to a no-op when testing other managers.
612 """
613 with TemporaryDirectory() as td:
614 with self.patch_cp_root(td):
615 self.test_checkpoints()
616
617 with TemporaryDirectory() as td:
618 with self.patch_cp_root(td):
619 self.test_file_checkpoints()
620
621
622 class GenericFileCheckpointsAPITest(APITest):
623 """
624 Run the tests from APITest with GenericFileCheckpoints.
625 """
626 config = Config()
627 config.FileContentsManager.checkpoints_class = GenericFileCheckpoints
628
629 def test_config_did_something(self):
630
631 self.assertIsInstance(
632 self.notebook.contents_manager.checkpoints,
633 GenericFileCheckpoints,
634 )
635
636
@@ -84,11 +84,16 b' class TestFileContentsManager(TestCase):'
84 root = td
84 root = td
85 os.mkdir(os.path.join(td, subd))
85 os.mkdir(os.path.join(td, subd))
86 fm = FileContentsManager(root_dir=root)
86 fm = FileContentsManager(root_dir=root)
87 cp_dir = fm.get_checkpoint_path('cp', 'test.ipynb')
87 cpm = fm.checkpoints
88 cp_subdir = fm.get_checkpoint_path('cp', '/%s/test.ipynb' % subd)
88 cp_dir = cpm.checkpoint_path(
89 'cp', 'test.ipynb'
90 )
91 cp_subdir = cpm.checkpoint_path(
92 'cp', '/%s/test.ipynb' % subd
93 )
89 self.assertNotEqual(cp_dir, cp_subdir)
94 self.assertNotEqual(cp_dir, cp_subdir)
90 self.assertEqual(cp_dir, os.path.join(root, fm.checkpoint_dir, cp_name))
95 self.assertEqual(cp_dir, os.path.join(root, cpm.checkpoint_dir, cp_name))
91 self.assertEqual(cp_subdir, os.path.join(root, subd, fm.checkpoint_dir, cp_name))
96 self.assertEqual(cp_subdir, os.path.join(root, subd, cpm.checkpoint_dir, cp_name))
92
97
93 @dec.skip_win32
98 @dec.skip_win32
94 def test_bad_symlink(self):
99 def test_bad_symlink(self):
General Comments 0
You need to be logged in to leave comments. Login now