Show More
@@ -0,0 +1,127 b'' | |||||
|
1 | """ | |||
|
2 | Classes for managing Checkpoints. | |||
|
3 | """ | |||
|
4 | ||||
|
5 | # Copyright (c) IPython Development Team. | |||
|
6 | # Distributed under the terms of the Modified BSD License. | |||
|
7 | ||||
|
8 | from tornado.web import HTTPError | |||
|
9 | ||||
|
10 | from IPython.config.configurable import LoggingConfigurable | |||
|
11 | ||||
|
12 | ||||
|
13 | class Checkpoints(LoggingConfigurable): | |||
|
14 | """ | |||
|
15 | Base class for managing checkpoints for a ContentsManager. | |||
|
16 | ||||
|
17 | Subclasses are required to implement: | |||
|
18 | ||||
|
19 | create_checkpoint(self, contents_mgr, path) | |||
|
20 | restore_checkpoint(self, contents_mgr, checkpoint_id, path) | |||
|
21 | rename_checkpoint(self, checkpoint_id, old_path, new_path) | |||
|
22 | delete_checkpoint(self, checkpoint_id, path) | |||
|
23 | list_checkpoints(self, path) | |||
|
24 | """ | |||
|
25 | def create_checkpoint(self, contents_mgr, path): | |||
|
26 | """Create a checkpoint.""" | |||
|
27 | raise NotImplementedError("must be implemented in a subclass") | |||
|
28 | ||||
|
29 | def restore_checkpoint(self, contents_mgr, checkpoint_id, path): | |||
|
30 | """Restore a checkpoint""" | |||
|
31 | raise NotImplementedError("must be implemented in a subclass") | |||
|
32 | ||||
|
33 | def rename_checkpoint(self, checkpoint_id, old_path, new_path): | |||
|
34 | """Rename a single checkpoint from old_path to new_path.""" | |||
|
35 | raise NotImplementedError("must be implemented in a subclass") | |||
|
36 | ||||
|
37 | def delete_checkpoint(self, checkpoint_id, path): | |||
|
38 | """delete a checkpoint for a file""" | |||
|
39 | raise NotImplementedError("must be implemented in a subclass") | |||
|
40 | ||||
|
41 | def list_checkpoints(self, path): | |||
|
42 | """Return a list of checkpoints for a given file""" | |||
|
43 | raise NotImplementedError("must be implemented in a subclass") | |||
|
44 | ||||
|
45 | def rename_all_checkpoints(self, old_path, new_path): | |||
|
46 | """Rename all checkpoints for old_path to new_path.""" | |||
|
47 | for cp in self.list_checkpoints(old_path): | |||
|
48 | self.rename_checkpoint(cp['id'], old_path, new_path) | |||
|
49 | ||||
|
50 | def delete_all_checkpoints(self, path): | |||
|
51 | """Delete all checkpoints for the given path.""" | |||
|
52 | for checkpoint in self.list_checkpoints(path): | |||
|
53 | self.delete_checkpoint(checkpoint['id'], path) | |||
|
54 | ||||
|
55 | ||||
|
56 | class GenericCheckpointsMixin(object): | |||
|
57 | """ | |||
|
58 | Helper for creating Checkpoints subclasses that can be used with any | |||
|
59 | ContentsManager. | |||
|
60 | ||||
|
61 | Provides a ContentsManager-agnostic implementation of `create_checkpoint` | |||
|
62 | and `restore_checkpoint` in terms of the following operations: | |||
|
63 | ||||
|
64 | - create_file_checkpoint(self, content, format, path) | |||
|
65 | - create_notebook_checkpoint(self, nb, path) | |||
|
66 | - get_file_checkpoint(self, checkpoint_id, path) | |||
|
67 | - get_notebook_checkpoint(self, checkpoint_id, path) | |||
|
68 | ||||
|
69 | To create a generic CheckpointManager, add this mixin to a class that | |||
|
70 | implement the above three methods plus the remaining Checkpoints API | |||
|
71 | methods: | |||
|
72 | ||||
|
73 | - delete_checkpoint(self, checkpoint_id, path) | |||
|
74 | - list_checkpoints(self, path) | |||
|
75 | - rename_checkpoint(self, checkpoint_id, old_path, new_path) | |||
|
76 | """ | |||
|
77 | ||||
|
78 | def create_checkpoint(self, contents_mgr, path): | |||
|
79 | model = contents_mgr.get(path, content=True) | |||
|
80 | type = model['type'] | |||
|
81 | if type == 'notebook': | |||
|
82 | return self.create_notebook_checkpoint( | |||
|
83 | model['content'], | |||
|
84 | path, | |||
|
85 | ) | |||
|
86 | elif type == 'file': | |||
|
87 | return self.create_file_checkpoint( | |||
|
88 | model['content'], | |||
|
89 | model['format'], | |||
|
90 | path, | |||
|
91 | ) | |||
|
92 | else: | |||
|
93 | raise HTTPError(500, u'Unexpected type %s' % type) | |||
|
94 | ||||
|
95 | def restore_checkpoint(self, contents_mgr, checkpoint_id, path): | |||
|
96 | """Restore a checkpoint.""" | |||
|
97 | type = contents_mgr.get(path, content=False)['type'] | |||
|
98 | if type == 'notebook': | |||
|
99 | model = self.get_notebook_checkpoint(checkpoint_id, path) | |||
|
100 | elif type == 'file': | |||
|
101 | model = self.get_file_checkpoint(checkpoint_id, path) | |||
|
102 | else: | |||
|
103 | raise HTTPError(500, u'Unexpected type %s' % type) | |||
|
104 | contents_mgr.save(model, path) | |||
|
105 | ||||
|
106 | # Required Methods | |||
|
107 | def create_file_checkpoint(self, content, format, path): | |||
|
108 | """Create a checkpoint of the current state of a file | |||
|
109 | ||||
|
110 | Returns a checkpoint model for the new checkpoint. | |||
|
111 | """ | |||
|
112 | raise NotImplementedError("must be implemented in a subclass") | |||
|
113 | ||||
|
114 | def create_notebook_checkpoint(self, nb, path): | |||
|
115 | """Create a checkpoint of the current state of a file | |||
|
116 | ||||
|
117 | Returns a checkpoint model for the new checkpoint. | |||
|
118 | """ | |||
|
119 | raise NotImplementedError("must be implemented in a subclass") | |||
|
120 | ||||
|
121 | def get_checkpoint(self, checkpoint_id, path, type): | |||
|
122 | """Get the content of a checkpoint. | |||
|
123 | ||||
|
124 | Returns an unvalidated model with the same structure as | |||
|
125 | the return value of ContentsManager.get | |||
|
126 | """ | |||
|
127 | raise NotImplementedError("must be implemented in a subclass") |
@@ -0,0 +1,200 b'' | |||||
|
1 | """ | |||
|
2 | File-based Checkpoints implementations. | |||
|
3 | """ | |||
|
4 | import os | |||
|
5 | import shutil | |||
|
6 | ||||
|
7 | from tornado.web import HTTPError | |||
|
8 | ||||
|
9 | from .checkpoints import ( | |||
|
10 | Checkpoints, | |||
|
11 | GenericCheckpointsMixin, | |||
|
12 | ) | |||
|
13 | from .fileio import FileManagerMixin | |||
|
14 | ||||
|
15 | from IPython.utils import tz | |||
|
16 | from IPython.utils.path import ensure_dir_exists | |||
|
17 | from IPython.utils.py3compat import getcwd | |||
|
18 | from IPython.utils.traitlets import Unicode | |||
|
19 | ||||
|
20 | ||||
|
21 | class FileCheckpoints(FileManagerMixin, Checkpoints): | |||
|
22 | """ | |||
|
23 | A Checkpoints that caches checkpoints for files in adjacent | |||
|
24 | directories. | |||
|
25 | ||||
|
26 | Only works with FileContentsManager. Use GenericFileCheckpoints if | |||
|
27 | you want file-based checkpoints with another ContentsManager. | |||
|
28 | """ | |||
|
29 | ||||
|
30 | checkpoint_dir = Unicode( | |||
|
31 | '.ipynb_checkpoints', | |||
|
32 | config=True, | |||
|
33 | help="""The directory name in which to keep file checkpoints | |||
|
34 | ||||
|
35 | This is a path relative to the file's own directory. | |||
|
36 | ||||
|
37 | By default, it is .ipynb_checkpoints | |||
|
38 | """, | |||
|
39 | ) | |||
|
40 | ||||
|
41 | root_dir = Unicode(config=True) | |||
|
42 | ||||
|
43 | def _root_dir_default(self): | |||
|
44 | try: | |||
|
45 | return self.parent.root_dir | |||
|
46 | except AttributeError: | |||
|
47 | return getcwd() | |||
|
48 | ||||
|
49 | # ContentsManager-dependent checkpoint API | |||
|
50 | def create_checkpoint(self, contents_mgr, path): | |||
|
51 | """Create a checkpoint.""" | |||
|
52 | checkpoint_id = u'checkpoint' | |||
|
53 | src_path = contents_mgr._get_os_path(path) | |||
|
54 | dest_path = self.checkpoint_path(checkpoint_id, path) | |||
|
55 | self._copy(src_path, dest_path) | |||
|
56 | return self.checkpoint_model(checkpoint_id, dest_path) | |||
|
57 | ||||
|
58 | def restore_checkpoint(self, contents_mgr, checkpoint_id, path): | |||
|
59 | """Restore a checkpoint.""" | |||
|
60 | src_path = self.checkpoint_path(checkpoint_id, path) | |||
|
61 | dest_path = contents_mgr._get_os_path(path) | |||
|
62 | self._copy(src_path, dest_path) | |||
|
63 | ||||
|
64 | # ContentsManager-independent checkpoint API | |||
|
65 | def rename_checkpoint(self, checkpoint_id, old_path, new_path): | |||
|
66 | """Rename a checkpoint from old_path to new_path.""" | |||
|
67 | old_cp_path = self.checkpoint_path(checkpoint_id, old_path) | |||
|
68 | new_cp_path = self.checkpoint_path(checkpoint_id, new_path) | |||
|
69 | if os.path.isfile(old_cp_path): | |||
|
70 | self.log.debug( | |||
|
71 | "Renaming checkpoint %s -> %s", | |||
|
72 | old_cp_path, | |||
|
73 | new_cp_path, | |||
|
74 | ) | |||
|
75 | with self.perm_to_403(): | |||
|
76 | shutil.move(old_cp_path, new_cp_path) | |||
|
77 | ||||
|
78 | def delete_checkpoint(self, checkpoint_id, path): | |||
|
79 | """delete a file's checkpoint""" | |||
|
80 | path = path.strip('/') | |||
|
81 | cp_path = self.checkpoint_path(checkpoint_id, path) | |||
|
82 | if not os.path.isfile(cp_path): | |||
|
83 | self.no_such_checkpoint(path, checkpoint_id) | |||
|
84 | ||||
|
85 | self.log.debug("unlinking %s", cp_path) | |||
|
86 | with self.perm_to_403(): | |||
|
87 | os.unlink(cp_path) | |||
|
88 | ||||
|
89 | def list_checkpoints(self, path): | |||
|
90 | """list the checkpoints for a given file | |||
|
91 | ||||
|
92 | This contents manager currently only supports one checkpoint per file. | |||
|
93 | """ | |||
|
94 | path = path.strip('/') | |||
|
95 | checkpoint_id = "checkpoint" | |||
|
96 | os_path = self.checkpoint_path(checkpoint_id, path) | |||
|
97 | if not os.path.isfile(os_path): | |||
|
98 | return [] | |||
|
99 | else: | |||
|
100 | return [self.checkpoint_model(checkpoint_id, os_path)] | |||
|
101 | ||||
|
102 | # Checkpoint-related utilities | |||
|
103 | def checkpoint_path(self, checkpoint_id, path): | |||
|
104 | """find the path to a checkpoint""" | |||
|
105 | path = path.strip('/') | |||
|
106 | parent, name = ('/' + path).rsplit('/', 1) | |||
|
107 | parent = parent.strip('/') | |||
|
108 | basename, ext = os.path.splitext(name) | |||
|
109 | filename = u"{name}-{checkpoint_id}{ext}".format( | |||
|
110 | name=basename, | |||
|
111 | checkpoint_id=checkpoint_id, | |||
|
112 | ext=ext, | |||
|
113 | ) | |||
|
114 | os_path = self._get_os_path(path=parent) | |||
|
115 | cp_dir = os.path.join(os_path, self.checkpoint_dir) | |||
|
116 | with self.perm_to_403(): | |||
|
117 | ensure_dir_exists(cp_dir) | |||
|
118 | cp_path = os.path.join(cp_dir, filename) | |||
|
119 | return cp_path | |||
|
120 | ||||
|
121 | def checkpoint_model(self, checkpoint_id, os_path): | |||
|
122 | """construct the info dict for a given checkpoint""" | |||
|
123 | stats = os.stat(os_path) | |||
|
124 | last_modified = tz.utcfromtimestamp(stats.st_mtime) | |||
|
125 | info = dict( | |||
|
126 | id=checkpoint_id, | |||
|
127 | last_modified=last_modified, | |||
|
128 | ) | |||
|
129 | return info | |||
|
130 | ||||
|
131 | # Error Handling | |||
|
132 | def no_such_checkpoint(self, path, checkpoint_id): | |||
|
133 | raise HTTPError( | |||
|
134 | 404, | |||
|
135 | u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id) | |||
|
136 | ) | |||
|
137 | ||||
|
138 | ||||
|
139 | class GenericFileCheckpoints(GenericCheckpointsMixin, FileCheckpoints): | |||
|
140 | """ | |||
|
141 | Local filesystem Checkpoints that works with any conforming | |||
|
142 | ContentsManager. | |||
|
143 | """ | |||
|
144 | def create_file_checkpoint(self, content, format, path): | |||
|
145 | """Create a checkpoint from the current content of a notebook.""" | |||
|
146 | path = path.strip('/') | |||
|
147 | # only the one checkpoint ID: | |||
|
148 | checkpoint_id = u"checkpoint" | |||
|
149 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |||
|
150 | self.log.debug("creating checkpoint for %s", path) | |||
|
151 | with self.perm_to_403(): | |||
|
152 | self._save_file(os_checkpoint_path, content, format=format) | |||
|
153 | ||||
|
154 | # return the checkpoint info | |||
|
155 | return self.checkpoint_model(checkpoint_id, os_checkpoint_path) | |||
|
156 | ||||
|
157 | def create_notebook_checkpoint(self, nb, path): | |||
|
158 | """Create a checkpoint from the current content of a notebook.""" | |||
|
159 | path = path.strip('/') | |||
|
160 | # only the one checkpoint ID: | |||
|
161 | checkpoint_id = u"checkpoint" | |||
|
162 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |||
|
163 | self.log.debug("creating checkpoint for %s", path) | |||
|
164 | with self.perm_to_403(): | |||
|
165 | self._save_notebook(os_checkpoint_path, nb) | |||
|
166 | ||||
|
167 | # return the checkpoint info | |||
|
168 | return self.checkpoint_model(checkpoint_id, os_checkpoint_path) | |||
|
169 | ||||
|
170 | def get_notebook_checkpoint(self, checkpoint_id, path): | |||
|
171 | ||||
|
172 | path = path.strip('/') | |||
|
173 | self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) | |||
|
174 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |||
|
175 | ||||
|
176 | if not os.path.isfile(os_checkpoint_path): | |||
|
177 | self.no_such_checkpoint(path, checkpoint_id) | |||
|
178 | ||||
|
179 | return { | |||
|
180 | 'type': 'notebook', | |||
|
181 | 'content': self._read_notebook( | |||
|
182 | os_checkpoint_path, | |||
|
183 | as_version=4, | |||
|
184 | ), | |||
|
185 | } | |||
|
186 | ||||
|
187 | def get_file_checkpoint(self, checkpoint_id, path): | |||
|
188 | path = path.strip('/') | |||
|
189 | self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) | |||
|
190 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |||
|
191 | ||||
|
192 | if not os.path.isfile(os_checkpoint_path): | |||
|
193 | self.no_such_checkpoint(path, checkpoint_id) | |||
|
194 | ||||
|
195 | content, format = self._read_file(os_checkpoint_path, format=None) | |||
|
196 | return { | |||
|
197 | 'type': 'file', | |||
|
198 | 'content': content, | |||
|
199 | 'format': format, | |||
|
200 | } |
@@ -0,0 +1,166 b'' | |||||
|
1 | """ | |||
|
2 | Utilities for file-based Contents/Checkpoints managers. | |||
|
3 | """ | |||
|
4 | ||||
|
5 | # Copyright (c) IPython Development Team. | |||
|
6 | # Distributed under the terms of the Modified BSD License. | |||
|
7 | ||||
|
8 | import base64 | |||
|
9 | from contextlib import contextmanager | |||
|
10 | import errno | |||
|
11 | import io | |||
|
12 | import os | |||
|
13 | import shutil | |||
|
14 | ||||
|
15 | from tornado.web import HTTPError | |||
|
16 | ||||
|
17 | from IPython.html.utils import ( | |||
|
18 | to_api_path, | |||
|
19 | to_os_path, | |||
|
20 | ) | |||
|
21 | from IPython import nbformat | |||
|
22 | from IPython.utils.io import atomic_writing | |||
|
23 | from IPython.utils.py3compat import str_to_unicode | |||
|
24 | ||||
|
25 | ||||
|
26 | class FileManagerMixin(object): | |||
|
27 | """ | |||
|
28 | Mixin for ContentsAPI classes that interact with the filesystem. | |||
|
29 | ||||
|
30 | Provides facilities for reading, writing, and copying both notebooks and | |||
|
31 | generic files. | |||
|
32 | ||||
|
33 | Shared by FileContentsManager and FileCheckpoints. | |||
|
34 | ||||
|
35 | Note | |||
|
36 | ---- | |||
|
37 | Classes using this mixin must provide the following attributes: | |||
|
38 | ||||
|
39 | root_dir : unicode | |||
|
40 | A directory against against which API-style paths are to be resolved. | |||
|
41 | ||||
|
42 | log : logging.Logger | |||
|
43 | """ | |||
|
44 | ||||
|
45 | @contextmanager | |||
|
46 | def open(self, os_path, *args, **kwargs): | |||
|
47 | """wrapper around io.open that turns permission errors into 403""" | |||
|
48 | with self.perm_to_403(os_path): | |||
|
49 | with io.open(os_path, *args, **kwargs) as f: | |||
|
50 | yield f | |||
|
51 | ||||
|
52 | @contextmanager | |||
|
53 | def atomic_writing(self, os_path, *args, **kwargs): | |||
|
54 | """wrapper around atomic_writing that turns permission errors to 403""" | |||
|
55 | with self.perm_to_403(os_path): | |||
|
56 | with atomic_writing(os_path, *args, **kwargs) as f: | |||
|
57 | yield f | |||
|
58 | ||||
|
59 | @contextmanager | |||
|
60 | def perm_to_403(self, os_path=''): | |||
|
61 | """context manager for turning permission errors into 403.""" | |||
|
62 | try: | |||
|
63 | yield | |||
|
64 | except OSError as e: | |||
|
65 | if e.errno in {errno.EPERM, errno.EACCES}: | |||
|
66 | # make 403 error message without root prefix | |||
|
67 | # this may not work perfectly on unicode paths on Python 2, | |||
|
68 | # but nobody should be doing that anyway. | |||
|
69 | if not os_path: | |||
|
70 | os_path = str_to_unicode(e.filename or 'unknown file') | |||
|
71 | path = to_api_path(os_path, root=self.root_dir) | |||
|
72 | raise HTTPError(403, u'Permission denied: %s' % path) | |||
|
73 | else: | |||
|
74 | raise | |||
|
75 | ||||
|
76 | def _copy(self, src, dest): | |||
|
77 | """copy src to dest | |||
|
78 | ||||
|
79 | like shutil.copy2, but log errors in copystat | |||
|
80 | """ | |||
|
81 | shutil.copyfile(src, dest) | |||
|
82 | try: | |||
|
83 | shutil.copystat(src, dest) | |||
|
84 | except OSError: | |||
|
85 | self.log.debug("copystat on %s failed", dest, exc_info=True) | |||
|
86 | ||||
|
87 | def _get_os_path(self, path): | |||
|
88 | """Given an API path, return its file system path. | |||
|
89 | ||||
|
90 | Parameters | |||
|
91 | ---------- | |||
|
92 | path : string | |||
|
93 | The relative API path to the named file. | |||
|
94 | ||||
|
95 | Returns | |||
|
96 | ------- | |||
|
97 | path : string | |||
|
98 | Native, absolute OS path to for a file. | |||
|
99 | """ | |||
|
100 | return to_os_path(path, self.root_dir) | |||
|
101 | ||||
|
102 | def _read_notebook(self, os_path, as_version=4): | |||
|
103 | """Read a notebook from an os path.""" | |||
|
104 | with self.open(os_path, 'r', encoding='utf-8') as f: | |||
|
105 | try: | |||
|
106 | return nbformat.read(f, as_version=as_version) | |||
|
107 | except Exception as e: | |||
|
108 | raise HTTPError( | |||
|
109 | 400, | |||
|
110 | u"Unreadable Notebook: %s %r" % (os_path, e), | |||
|
111 | ) | |||
|
112 | ||||
|
113 | def _save_notebook(self, os_path, nb): | |||
|
114 | """Save a notebook to an os_path.""" | |||
|
115 | with self.atomic_writing(os_path, encoding='utf-8') as f: | |||
|
116 | nbformat.write(nb, f, version=nbformat.NO_CONVERT) | |||
|
117 | ||||
|
118 | def _read_file(self, os_path, format): | |||
|
119 | """Read a non-notebook file. | |||
|
120 | ||||
|
121 | os_path: The path to be read. | |||
|
122 | format: | |||
|
123 | If 'text', the contents will be decoded as UTF-8. | |||
|
124 | If 'base64', the raw bytes contents will be encoded as base64. | |||
|
125 | If not specified, try to decode as UTF-8, and fall back to base64 | |||
|
126 | """ | |||
|
127 | if not os.path.isfile(os_path): | |||
|
128 | raise HTTPError(400, "Cannot read non-file %s" % os_path) | |||
|
129 | ||||
|
130 | with self.open(os_path, 'rb') as f: | |||
|
131 | bcontent = f.read() | |||
|
132 | ||||
|
133 | if format is None or format == 'text': | |||
|
134 | # Try to interpret as unicode if format is unknown or if unicode | |||
|
135 | # was explicitly requested. | |||
|
136 | try: | |||
|
137 | return bcontent.decode('utf8'), 'text' | |||
|
138 | except UnicodeError: | |||
|
139 | if format == 'text': | |||
|
140 | raise HTTPError( | |||
|
141 | 400, | |||
|
142 | "%s is not UTF-8 encoded" % os_path, | |||
|
143 | reason='bad format', | |||
|
144 | ) | |||
|
145 | return base64.encodestring(bcontent).decode('ascii'), 'base64' | |||
|
146 | ||||
|
147 | def _save_file(self, os_path, content, format): | |||
|
148 | """Save content of a generic file.""" | |||
|
149 | if format not in {'text', 'base64'}: | |||
|
150 | raise HTTPError( | |||
|
151 | 400, | |||
|
152 | "Must specify format of file contents as 'text' or 'base64'", | |||
|
153 | ) | |||
|
154 | try: | |||
|
155 | if format == 'text': | |||
|
156 | bcontent = content.encode('utf8') | |||
|
157 | else: | |||
|
158 | b64_bytes = content.encode('ascii') | |||
|
159 | bcontent = base64.decodestring(b64_bytes) | |||
|
160 | except Exception as e: | |||
|
161 | raise HTTPError( | |||
|
162 | 400, u'Encoding error saving %s: %s' % (os_path, e) | |||
|
163 | ) | |||
|
164 | ||||
|
165 | with self.atomic_writing(os_path, text=False) as f: | |||
|
166 | f.write(bcontent) |
@@ -3,28 +3,31 b'' | |||||
3 | # Copyright (c) IPython Development Team. |
|
3 | # Copyright (c) IPython Development Team. | |
4 | # Distributed under the terms of the Modified BSD License. |
|
4 | # Distributed under the terms of the Modified BSD License. | |
5 |
|
5 | |||
6 | import base64 |
|
6 | ||
7 | import errno |
|
|||
8 | import io |
|
7 | import io | |
9 | import os |
|
8 | import os | |
10 | import shutil |
|
9 | import shutil | |
11 | from contextlib import contextmanager |
|
|||
12 | import mimetypes |
|
10 | import mimetypes | |
13 |
|
11 | |||
14 | from tornado import web |
|
12 | from tornado import web | |
15 |
|
13 | |||
|
14 | from .filecheckpoints import FileCheckpoints | |||
|
15 | from .fileio import FileManagerMixin | |||
16 | from .manager import ContentsManager |
|
16 | from .manager import ContentsManager | |
|
17 | ||||
17 | from IPython import nbformat |
|
18 | from IPython import nbformat | |
18 | from IPython.utils.io import atomic_writing |
|
|||
19 | from IPython.utils.importstring import import_item |
|
19 | from IPython.utils.importstring import import_item | |
20 | from IPython.utils.path import ensure_dir_exists |
|
|||
21 | from IPython.utils.traitlets import Any, Unicode, Bool, TraitError |
|
20 | from IPython.utils.traitlets import Any, Unicode, Bool, TraitError | |
22 |
from IPython.utils.py3compat import getcwd, |
|
21 | from IPython.utils.py3compat import getcwd, string_types | |
23 | from IPython.utils import tz |
|
22 | from IPython.utils import tz | |
24 |
from IPython.html.utils import |
|
23 | from IPython.html.utils import ( | |
|
24 | is_hidden, | |||
|
25 | to_api_path, | |||
|
26 | ) | |||
25 |
|
27 | |||
26 | _script_exporter = None |
|
28 | _script_exporter = None | |
27 |
|
29 | |||
|
30 | ||||
28 | def _post_save_script(model, os_path, contents_manager, **kwargs): |
|
31 | def _post_save_script(model, os_path, contents_manager, **kwargs): | |
29 | """convert notebooks to Python script after save with nbconvert |
|
32 | """convert notebooks to Python script after save with nbconvert | |
30 |
|
33 | |||
@@ -48,7 +51,8 b' def _post_save_script(model, os_path, contents_manager, **kwargs):' | |||||
48 | with io.open(script_fname, 'w', encoding='utf-8') as f: |
|
51 | with io.open(script_fname, 'w', encoding='utf-8') as f: | |
49 | f.write(script) |
|
52 | f.write(script) | |
50 |
|
53 | |||
51 | class FileContentsManager(ContentsManager): |
|
54 | ||
|
55 | class FileContentsManager(FileManagerMixin, ContentsManager): | |||
52 |
|
56 | |||
53 | root_dir = Unicode(config=True) |
|
57 | root_dir = Unicode(config=True) | |
54 |
|
58 | |||
@@ -57,38 +61,7 b' class FileContentsManager(ContentsManager):' | |||||
57 | return self.parent.notebook_dir |
|
61 | return self.parent.notebook_dir | |
58 | except AttributeError: |
|
62 | except AttributeError: | |
59 | return getcwd() |
|
63 | return getcwd() | |
60 |
|
64 | |||
61 | @contextmanager |
|
|||
62 | def perm_to_403(self, os_path=''): |
|
|||
63 | """context manager for turning permission errors into 403""" |
|
|||
64 | try: |
|
|||
65 | yield |
|
|||
66 | except OSError as e: |
|
|||
67 | if e.errno in {errno.EPERM, errno.EACCES}: |
|
|||
68 | # make 403 error message without root prefix |
|
|||
69 | # this may not work perfectly on unicode paths on Python 2, |
|
|||
70 | # but nobody should be doing that anyway. |
|
|||
71 | if not os_path: |
|
|||
72 | os_path = str_to_unicode(e.filename or 'unknown file') |
|
|||
73 | path = to_api_path(os_path, self.root_dir) |
|
|||
74 | raise web.HTTPError(403, u'Permission denied: %s' % path) |
|
|||
75 | else: |
|
|||
76 | raise |
|
|||
77 |
|
||||
78 | @contextmanager |
|
|||
79 | def open(self, os_path, *args, **kwargs): |
|
|||
80 | """wrapper around io.open that turns permission errors into 403""" |
|
|||
81 | with self.perm_to_403(os_path): |
|
|||
82 | with io.open(os_path, *args, **kwargs) as f: |
|
|||
83 | yield f |
|
|||
84 |
|
||||
85 | @contextmanager |
|
|||
86 | def atomic_writing(self, os_path, *args, **kwargs): |
|
|||
87 | """wrapper around atomic_writing that turns permission errors into 403""" |
|
|||
88 | with self.perm_to_403(os_path): |
|
|||
89 | with atomic_writing(os_path, *args, **kwargs) as f: |
|
|||
90 | yield f |
|
|||
91 |
|
||||
92 | save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook') |
|
65 | save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook') | |
93 | def _save_script_changed(self): |
|
66 | def _save_script_changed(self): | |
94 | self.log.warn(""" |
|
67 | self.log.warn(""" | |
@@ -148,60 +121,8 b' class FileContentsManager(ContentsManager):' | |||||
148 | if not os.path.isdir(new): |
|
121 | if not os.path.isdir(new): | |
149 | raise TraitError("%r is not a directory" % new) |
|
122 | raise TraitError("%r is not a directory" % new) | |
150 |
|
123 | |||
151 | checkpoint_dir = Unicode('.ipynb_checkpoints', config=True, |
|
124 | def _checkpoints_class_default(self): | |
152 | help="""The directory name in which to keep file checkpoints |
|
125 | return FileCheckpoints | |
153 |
|
||||
154 | This is a path relative to the file's own directory. |
|
|||
155 |
|
||||
156 | By default, it is .ipynb_checkpoints |
|
|||
157 | """ |
|
|||
158 | ) |
|
|||
159 |
|
||||
160 | def _copy(self, src, dest): |
|
|||
161 | """copy src to dest |
|
|||
162 |
|
||||
163 | like shutil.copy2, but log errors in copystat |
|
|||
164 | """ |
|
|||
165 | shutil.copyfile(src, dest) |
|
|||
166 | try: |
|
|||
167 | shutil.copystat(src, dest) |
|
|||
168 | except OSError as e: |
|
|||
169 | self.log.debug("copystat on %s failed", dest, exc_info=True) |
|
|||
170 |
|
||||
171 | def _get_os_path(self, path): |
|
|||
172 | """Given an API path, return its file system path. |
|
|||
173 |
|
||||
174 | Parameters |
|
|||
175 | ---------- |
|
|||
176 | path : string |
|
|||
177 | The relative API path to the named file. |
|
|||
178 |
|
||||
179 | Returns |
|
|||
180 | ------- |
|
|||
181 | path : string |
|
|||
182 | Native, absolute OS path to for a file. |
|
|||
183 | """ |
|
|||
184 | return to_os_path(path, self.root_dir) |
|
|||
185 |
|
||||
186 | def dir_exists(self, path): |
|
|||
187 | """Does the API-style path refer to an extant directory? |
|
|||
188 |
|
||||
189 | API-style wrapper for os.path.isdir |
|
|||
190 |
|
||||
191 | Parameters |
|
|||
192 | ---------- |
|
|||
193 | path : string |
|
|||
194 | The path to check. This is an API path (`/` separated, |
|
|||
195 | relative to root_dir). |
|
|||
196 |
|
||||
197 | Returns |
|
|||
198 | ------- |
|
|||
199 | exists : bool |
|
|||
200 | Whether the path is indeed a directory. |
|
|||
201 | """ |
|
|||
202 | path = path.strip('/') |
|
|||
203 | os_path = self._get_os_path(path=path) |
|
|||
204 | return os.path.isdir(os_path) |
|
|||
205 |
|
126 | |||
206 | def is_hidden(self, path): |
|
127 | def is_hidden(self, path): | |
207 | """Does the API style path correspond to a hidden directory or file? |
|
128 | """Does the API style path correspond to a hidden directory or file? | |
@@ -240,6 +161,26 b' class FileContentsManager(ContentsManager):' | |||||
240 | os_path = self._get_os_path(path) |
|
161 | os_path = self._get_os_path(path) | |
241 | return os.path.isfile(os_path) |
|
162 | return os.path.isfile(os_path) | |
242 |
|
163 | |||
|
164 | def dir_exists(self, path): | |||
|
165 | """Does the API-style path refer to an extant directory? | |||
|
166 | ||||
|
167 | API-style wrapper for os.path.isdir | |||
|
168 | ||||
|
169 | Parameters | |||
|
170 | ---------- | |||
|
171 | path : string | |||
|
172 | The path to check. This is an API path (`/` separated, | |||
|
173 | relative to root_dir). | |||
|
174 | ||||
|
175 | Returns | |||
|
176 | ------- | |||
|
177 | exists : bool | |||
|
178 | Whether the path is indeed a directory. | |||
|
179 | """ | |||
|
180 | path = path.strip('/') | |||
|
181 | os_path = self._get_os_path(path=path) | |||
|
182 | return os.path.isdir(os_path) | |||
|
183 | ||||
243 | def exists(self, path): |
|
184 | def exists(self, path): | |
244 | """Returns True if the path exists, else returns False. |
|
185 | """Returns True if the path exists, else returns False. | |
245 |
|
186 | |||
@@ -338,33 +279,20 b' class FileContentsManager(ContentsManager):' | |||||
338 | os_path = self._get_os_path(path) |
|
279 | os_path = self._get_os_path(path) | |
339 |
|
280 | |||
340 | if content: |
|
281 | if content: | |
341 |
|
|
282 | content, format = self._read_file(os_path, format) | |
342 | # could be FIFO |
|
283 | default_mime = { | |
343 | raise web.HTTPError(400, "Cannot get content of non-file %s" % os_path) |
|
284 | 'text': 'text/plain', | |
344 | with self.open(os_path, 'rb') as f: |
|
285 | 'base64': 'application/octet-stream' | |
345 | bcontent = f.read() |
|
286 | }[format] | |
346 |
|
287 | |||
347 | if format != 'base64': |
|
288 | model.update( | |
348 |
|
|
289 | content=content, | |
349 | model['content'] = bcontent.decode('utf8') |
|
290 | format=format, | |
350 | except UnicodeError as e: |
|
291 | mimetype=mimetypes.guess_type(os_path)[0] or default_mime, | |
351 | if format == 'text': |
|
292 | ) | |
352 | raise web.HTTPError(400, "%s is not UTF-8 encoded" % path, reason='bad format') |
|
|||
353 | else: |
|
|||
354 | model['format'] = 'text' |
|
|||
355 | default_mime = 'text/plain' |
|
|||
356 |
|
||||
357 | if model['content'] is None: |
|
|||
358 | model['content'] = base64.encodestring(bcontent).decode('ascii') |
|
|||
359 | model['format'] = 'base64' |
|
|||
360 | if model['format'] == 'base64': |
|
|||
361 | default_mime = 'application/octet-stream' |
|
|||
362 |
|
||||
363 | model['mimetype'] = mimetypes.guess_type(os_path)[0] or default_mime |
|
|||
364 |
|
293 | |||
365 | return model |
|
294 | return model | |
366 |
|
295 | |||
367 |
|
||||
368 | def _notebook_model(self, path, content=True): |
|
296 | def _notebook_model(self, path, content=True): | |
369 | """Build a notebook model |
|
297 | """Build a notebook model | |
370 |
|
298 | |||
@@ -375,11 +303,7 b' class FileContentsManager(ContentsManager):' | |||||
375 | model['type'] = 'notebook' |
|
303 | model['type'] = 'notebook' | |
376 | if content: |
|
304 | if content: | |
377 | os_path = self._get_os_path(path) |
|
305 | os_path = self._get_os_path(path) | |
378 | with self.open(os_path, 'r', encoding='utf-8') as f: |
|
306 | nb = self._read_notebook(os_path, as_version=4) | |
379 | try: |
|
|||
380 | nb = nbformat.read(f, as_version=4) |
|
|||
381 | except Exception as e: |
|
|||
382 | raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e)) |
|
|||
383 | self.mark_trusted_cells(nb, path) |
|
307 | self.mark_trusted_cells(nb, path) | |
384 | model['content'] = nb |
|
308 | model['content'] = nb | |
385 | model['format'] = 'json' |
|
309 | model['format'] = 'json' | |
@@ -428,33 +352,6 b' class FileContentsManager(ContentsManager):' | |||||
428 | model = self._file_model(path, content=content, format=format) |
|
352 | model = self._file_model(path, content=content, format=format) | |
429 | return model |
|
353 | return model | |
430 |
|
354 | |||
431 | def _save_notebook(self, os_path, model, path=''): |
|
|||
432 | """save a notebook file""" |
|
|||
433 | # Save the notebook file |
|
|||
434 | nb = nbformat.from_dict(model['content']) |
|
|||
435 |
|
||||
436 | self.check_and_sign(nb, path) |
|
|||
437 |
|
||||
438 | with self.atomic_writing(os_path, encoding='utf-8') as f: |
|
|||
439 | nbformat.write(nb, f, version=nbformat.NO_CONVERT) |
|
|||
440 |
|
||||
441 | def _save_file(self, os_path, model, path=''): |
|
|||
442 | """save a non-notebook file""" |
|
|||
443 | fmt = model.get('format', None) |
|
|||
444 | if fmt not in {'text', 'base64'}: |
|
|||
445 | raise web.HTTPError(400, "Must specify format of file contents as 'text' or 'base64'") |
|
|||
446 | try: |
|
|||
447 | content = model['content'] |
|
|||
448 | if fmt == 'text': |
|
|||
449 | bcontent = content.encode('utf8') |
|
|||
450 | else: |
|
|||
451 | b64_bytes = content.encode('ascii') |
|
|||
452 | bcontent = base64.decodestring(b64_bytes) |
|
|||
453 | except Exception as e: |
|
|||
454 | raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e)) |
|
|||
455 | with self.atomic_writing(os_path, text=False) as f: |
|
|||
456 | f.write(bcontent) |
|
|||
457 |
|
||||
458 | def _save_directory(self, os_path, model, path=''): |
|
355 | def _save_directory(self, os_path, model, path=''): | |
459 | """create a directory""" |
|
356 | """create a directory""" | |
460 | if is_hidden(os_path, self.root_dir): |
|
357 | if is_hidden(os_path, self.root_dir): | |
@@ -478,17 +375,19 b' class FileContentsManager(ContentsManager):' | |||||
478 |
|
375 | |||
479 | self.run_pre_save_hook(model=model, path=path) |
|
376 | self.run_pre_save_hook(model=model, path=path) | |
480 |
|
377 | |||
481 | # One checkpoint should always exist |
|
|||
482 | if self.file_exists(path) and not self.list_checkpoints(path): |
|
|||
483 | self.create_checkpoint(path) |
|
|||
484 |
|
||||
485 | os_path = self._get_os_path(path) |
|
378 | os_path = self._get_os_path(path) | |
486 | self.log.debug("Saving %s", os_path) |
|
379 | self.log.debug("Saving %s", os_path) | |
487 | try: |
|
380 | try: | |
488 | if model['type'] == 'notebook': |
|
381 | if model['type'] == 'notebook': | |
489 | self._save_notebook(os_path, model, path) |
|
382 | nb = nbformat.from_dict(model['content']) | |
|
383 | self.check_and_sign(nb, path) | |||
|
384 | self._save_notebook(os_path, nb) | |||
|
385 | # One checkpoint should always exist for notebooks. | |||
|
386 | if not self.checkpoints.list_checkpoints(path): | |||
|
387 | self.create_checkpoint(path) | |||
490 | elif model['type'] == 'file': |
|
388 | elif model['type'] == 'file': | |
491 | self._save_file(os_path, model, path) |
|
389 | # Missing format will be handled internally by _save_file. | |
|
390 | self._save_file(os_path, model['content'], model.get('format')) | |||
492 | elif model['type'] == 'directory': |
|
391 | elif model['type'] == 'directory': | |
493 | self._save_directory(os_path, model, path) |
|
392 | self._save_directory(os_path, model, path) | |
494 | else: |
|
393 | else: | |
@@ -512,28 +411,23 b' class FileContentsManager(ContentsManager):' | |||||
512 |
|
411 | |||
513 | return model |
|
412 | return model | |
514 |
|
413 | |||
515 | def delete(self, path): |
|
414 | def delete_file(self, path): | |
516 | """Delete file at path.""" |
|
415 | """Delete file at path.""" | |
517 | path = path.strip('/') |
|
416 | path = path.strip('/') | |
518 | os_path = self._get_os_path(path) |
|
417 | os_path = self._get_os_path(path) | |
519 | rm = os.unlink |
|
418 | rm = os.unlink | |
520 | if os.path.isdir(os_path): |
|
419 | if os.path.isdir(os_path): | |
521 | listing = os.listdir(os_path) |
|
420 | listing = os.listdir(os_path) | |
522 |
# |
|
421 | # Don't delete non-empty directories. | |
523 | if listing and listing != [self.checkpoint_dir]: |
|
422 | # A directory containing only leftover checkpoints is | |
524 | raise web.HTTPError(400, u'Directory %s not empty' % os_path) |
|
423 | # considered empty. | |
|
424 | cp_dir = getattr(self.checkpoints, 'checkpoint_dir', None) | |||
|
425 | for entry in listing: | |||
|
426 | if entry != cp_dir: | |||
|
427 | raise web.HTTPError(400, u'Directory %s not empty' % os_path) | |||
525 | elif not os.path.isfile(os_path): |
|
428 | elif not os.path.isfile(os_path): | |
526 | raise web.HTTPError(404, u'File does not exist: %s' % os_path) |
|
429 | raise web.HTTPError(404, u'File does not exist: %s' % os_path) | |
527 |
|
430 | |||
528 | # clear checkpoints |
|
|||
529 | for checkpoint in self.list_checkpoints(path): |
|
|||
530 | checkpoint_id = checkpoint['id'] |
|
|||
531 | cp_path = self.get_checkpoint_path(checkpoint_id, path) |
|
|||
532 | if os.path.isfile(cp_path): |
|
|||
533 | self.log.debug("Unlinking checkpoint %s", cp_path) |
|
|||
534 | with self.perm_to_403(): |
|
|||
535 | rm(cp_path) |
|
|||
536 |
|
||||
537 | if os.path.isdir(os_path): |
|
431 | if os.path.isdir(os_path): | |
538 | self.log.debug("Removing directory %s", os_path) |
|
432 | self.log.debug("Removing directory %s", os_path) | |
539 | with self.perm_to_403(): |
|
433 | with self.perm_to_403(): | |
@@ -543,7 +437,7 b' class FileContentsManager(ContentsManager):' | |||||
543 | with self.perm_to_403(): |
|
437 | with self.perm_to_403(): | |
544 | rm(os_path) |
|
438 | rm(os_path) | |
545 |
|
439 | |||
546 | def rename(self, old_path, new_path): |
|
440 | def rename_file(self, old_path, new_path): | |
547 | """Rename a file.""" |
|
441 | """Rename a file.""" | |
548 | old_path = old_path.strip('/') |
|
442 | old_path = old_path.strip('/') | |
549 | new_path = new_path.strip('/') |
|
443 | new_path = new_path.strip('/') | |
@@ -566,111 +460,6 b' class FileContentsManager(ContentsManager):' | |||||
566 | except Exception as e: |
|
460 | except Exception as e: | |
567 | raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e)) |
|
461 | raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e)) | |
568 |
|
462 | |||
569 | # Move the checkpoints |
|
|||
570 | old_checkpoints = self.list_checkpoints(old_path) |
|
|||
571 | for cp in old_checkpoints: |
|
|||
572 | checkpoint_id = cp['id'] |
|
|||
573 | old_cp_path = self.get_checkpoint_path(checkpoint_id, old_path) |
|
|||
574 | new_cp_path = self.get_checkpoint_path(checkpoint_id, new_path) |
|
|||
575 | if os.path.isfile(old_cp_path): |
|
|||
576 | self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path) |
|
|||
577 | with self.perm_to_403(): |
|
|||
578 | shutil.move(old_cp_path, new_cp_path) |
|
|||
579 |
|
||||
580 | # Checkpoint-related utilities |
|
|||
581 |
|
||||
582 | def get_checkpoint_path(self, checkpoint_id, path): |
|
|||
583 | """find the path to a checkpoint""" |
|
|||
584 | path = path.strip('/') |
|
|||
585 | parent, name = ('/' + path).rsplit('/', 1) |
|
|||
586 | parent = parent.strip('/') |
|
|||
587 | basename, ext = os.path.splitext(name) |
|
|||
588 | filename = u"{name}-{checkpoint_id}{ext}".format( |
|
|||
589 | name=basename, |
|
|||
590 | checkpoint_id=checkpoint_id, |
|
|||
591 | ext=ext, |
|
|||
592 | ) |
|
|||
593 | os_path = self._get_os_path(path=parent) |
|
|||
594 | cp_dir = os.path.join(os_path, self.checkpoint_dir) |
|
|||
595 | with self.perm_to_403(): |
|
|||
596 | ensure_dir_exists(cp_dir) |
|
|||
597 | cp_path = os.path.join(cp_dir, filename) |
|
|||
598 | return cp_path |
|
|||
599 |
|
||||
600 | def get_checkpoint_model(self, checkpoint_id, path): |
|
|||
601 | """construct the info dict for a given checkpoint""" |
|
|||
602 | path = path.strip('/') |
|
|||
603 | cp_path = self.get_checkpoint_path(checkpoint_id, path) |
|
|||
604 | stats = os.stat(cp_path) |
|
|||
605 | last_modified = tz.utcfromtimestamp(stats.st_mtime) |
|
|||
606 | info = dict( |
|
|||
607 | id = checkpoint_id, |
|
|||
608 | last_modified = last_modified, |
|
|||
609 | ) |
|
|||
610 | return info |
|
|||
611 |
|
||||
612 | # public checkpoint API |
|
|||
613 |
|
||||
614 | def create_checkpoint(self, path): |
|
|||
615 | """Create a checkpoint from the current state of a file""" |
|
|||
616 | path = path.strip('/') |
|
|||
617 | if not self.file_exists(path): |
|
|||
618 | raise web.HTTPError(404) |
|
|||
619 | src_path = self._get_os_path(path) |
|
|||
620 | # only the one checkpoint ID: |
|
|||
621 | checkpoint_id = u"checkpoint" |
|
|||
622 | cp_path = self.get_checkpoint_path(checkpoint_id, path) |
|
|||
623 | self.log.debug("creating checkpoint for %s", path) |
|
|||
624 | with self.perm_to_403(): |
|
|||
625 | self._copy(src_path, cp_path) |
|
|||
626 |
|
||||
627 | # return the checkpoint info |
|
|||
628 | return self.get_checkpoint_model(checkpoint_id, path) |
|
|||
629 |
|
||||
630 | def list_checkpoints(self, path): |
|
|||
631 | """list the checkpoints for a given file |
|
|||
632 |
|
||||
633 | This contents manager currently only supports one checkpoint per file. |
|
|||
634 | """ |
|
|||
635 | path = path.strip('/') |
|
|||
636 | checkpoint_id = "checkpoint" |
|
|||
637 | os_path = self.get_checkpoint_path(checkpoint_id, path) |
|
|||
638 | if not os.path.exists(os_path): |
|
|||
639 | return [] |
|
|||
640 | else: |
|
|||
641 | return [self.get_checkpoint_model(checkpoint_id, path)] |
|
|||
642 |
|
||||
643 |
|
||||
644 | def restore_checkpoint(self, checkpoint_id, path): |
|
|||
645 | """restore a file to a checkpointed state""" |
|
|||
646 | path = path.strip('/') |
|
|||
647 | self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) |
|
|||
648 | nb_path = self._get_os_path(path) |
|
|||
649 | cp_path = self.get_checkpoint_path(checkpoint_id, path) |
|
|||
650 | if not os.path.isfile(cp_path): |
|
|||
651 | self.log.debug("checkpoint file does not exist: %s", cp_path) |
|
|||
652 | raise web.HTTPError(404, |
|
|||
653 | u'checkpoint does not exist: %s@%s' % (path, checkpoint_id) |
|
|||
654 | ) |
|
|||
655 | # ensure notebook is readable (never restore from an unreadable notebook) |
|
|||
656 | if cp_path.endswith('.ipynb'): |
|
|||
657 | with self.open(cp_path, 'r', encoding='utf-8') as f: |
|
|||
658 | nbformat.read(f, as_version=4) |
|
|||
659 | self.log.debug("copying %s -> %s", cp_path, nb_path) |
|
|||
660 | with self.perm_to_403(): |
|
|||
661 | self._copy(cp_path, nb_path) |
|
|||
662 |
|
||||
663 | def delete_checkpoint(self, checkpoint_id, path): |
|
|||
664 | """delete a file's checkpoint""" |
|
|||
665 | path = path.strip('/') |
|
|||
666 | cp_path = self.get_checkpoint_path(checkpoint_id, path) |
|
|||
667 | if not os.path.isfile(cp_path): |
|
|||
668 | raise web.HTTPError(404, |
|
|||
669 | u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id) |
|
|||
670 | ) |
|
|||
671 | self.log.debug("unlinking %s", cp_path) |
|
|||
672 | os.unlink(cp_path) |
|
|||
673 |
|
||||
674 | def info_string(self): |
|
463 | def info_string(self): | |
675 | return "Serving notebooks from local directory: %s" % self.root_dir |
|
464 | return "Serving notebooks from local directory: %s" % self.root_dir | |
676 |
|
465 |
@@ -11,15 +11,25 b' import re' | |||||
11 |
|
11 | |||
12 | from tornado.web import HTTPError |
|
12 | from tornado.web import HTTPError | |
13 |
|
13 | |||
|
14 | from .checkpoints import Checkpoints | |||
14 | from IPython.config.configurable import LoggingConfigurable |
|
15 | from IPython.config.configurable import LoggingConfigurable | |
15 | from IPython.nbformat import sign, validate, ValidationError |
|
16 | from IPython.nbformat import sign, validate, ValidationError | |
16 | from IPython.nbformat.v4 import new_notebook |
|
17 | from IPython.nbformat.v4 import new_notebook | |
17 | from IPython.utils.importstring import import_item |
|
18 | from IPython.utils.importstring import import_item | |
18 |
from IPython.utils.traitlets import |
|
19 | from IPython.utils.traitlets import ( | |
|
20 | Any, | |||
|
21 | Dict, | |||
|
22 | Instance, | |||
|
23 | List, | |||
|
24 | TraitError, | |||
|
25 | Type, | |||
|
26 | Unicode, | |||
|
27 | ) | |||
19 | from IPython.utils.py3compat import string_types |
|
28 | from IPython.utils.py3compat import string_types | |
20 |
|
29 | |||
21 | copy_pat = re.compile(r'\-Copy\d*\.') |
|
30 | copy_pat = re.compile(r'\-Copy\d*\.') | |
22 |
|
31 | |||
|
32 | ||||
23 | class ContentsManager(LoggingConfigurable): |
|
33 | class ContentsManager(LoggingConfigurable): | |
24 | """Base class for serving files and directories. |
|
34 | """Base class for serving files and directories. | |
25 |
|
35 | |||
@@ -97,6 +107,19 b' class ContentsManager(LoggingConfigurable):' | |||||
97 | except Exception: |
|
107 | except Exception: | |
98 | self.log.error("Pre-save hook failed on %s", path, exc_info=True) |
|
108 | self.log.error("Pre-save hook failed on %s", path, exc_info=True) | |
99 |
|
109 | |||
|
110 | checkpoints_class = Type(Checkpoints, config=True) | |||
|
111 | checkpoints = Instance(Checkpoints, config=True) | |||
|
112 | checkpoints_kwargs = Dict(allow_none=False, config=True) | |||
|
113 | ||||
|
114 | def _checkpoints_default(self): | |||
|
115 | return self.checkpoints_class(**self.checkpoints_kwargs) | |||
|
116 | ||||
|
117 | def _checkpoints_kwargs_default(self): | |||
|
118 | return dict( | |||
|
119 | parent=self, | |||
|
120 | log=self.log, | |||
|
121 | ) | |||
|
122 | ||||
100 | # ContentsManager API part 1: methods that must be |
|
123 | # ContentsManager API part 1: methods that must be | |
101 | # implemented in subclasses. |
|
124 | # implemented in subclasses. | |
102 |
|
125 | |||
@@ -186,32 +209,27 b' class ContentsManager(LoggingConfigurable):' | |||||
186 | """ |
|
209 | """ | |
187 | raise NotImplementedError('must be implemented in a subclass') |
|
210 | raise NotImplementedError('must be implemented in a subclass') | |
188 |
|
211 | |||
189 | def delete(self, path): |
|
212 | def delete_file(self, path): | |
190 | """Delete file or directory by path.""" |
|
213 | """Delete file or directory by path.""" | |
191 | raise NotImplementedError('must be implemented in a subclass') |
|
214 | raise NotImplementedError('must be implemented in a subclass') | |
192 |
|
215 | |||
193 |
def |
|
216 | def rename_file(self, old_path, new_path): | |
194 | """Create a checkpoint of the current state of a file |
|
217 | """Rename a file.""" | |
195 |
|
218 | raise NotImplementedError('must be implemented in a subclass') | ||
196 | Returns a checkpoint_id for the new checkpoint. |
|
|||
197 | """ |
|
|||
198 | raise NotImplementedError("must be implemented in a subclass") |
|
|||
199 |
|
||||
200 | def list_checkpoints(self, path): |
|
|||
201 | """Return a list of checkpoints for a given file""" |
|
|||
202 | return [] |
|
|||
203 |
|
||||
204 | def restore_checkpoint(self, checkpoint_id, path): |
|
|||
205 | """Restore a file from one of its checkpoints""" |
|
|||
206 | raise NotImplementedError("must be implemented in a subclass") |
|
|||
207 |
|
||||
208 | def delete_checkpoint(self, checkpoint_id, path): |
|
|||
209 | """delete a checkpoint for a file""" |
|
|||
210 | raise NotImplementedError("must be implemented in a subclass") |
|
|||
211 |
|
219 | |||
212 | # ContentsManager API part 2: methods that have useable default |
|
220 | # ContentsManager API part 2: methods that have useable default | |
213 | # implementations, but can be overridden in subclasses. |
|
221 | # implementations, but can be overridden in subclasses. | |
214 |
|
222 | |||
|
223 | def delete(self, path): | |||
|
224 | """Delete a file/directory and any associated checkpoints.""" | |||
|
225 | self.delete_file(path) | |||
|
226 | self.checkpoints.delete_all_checkpoints(path) | |||
|
227 | ||||
|
228 | def rename(self, old_path, new_path): | |||
|
229 | """Rename a file and any checkpoints associated with that file.""" | |||
|
230 | self.rename_file(old_path, new_path) | |||
|
231 | self.checkpoints.rename_all_checkpoints(old_path, new_path) | |||
|
232 | ||||
215 | def update(self, model, path): |
|
233 | def update(self, model, path): | |
216 | """Update the file's path |
|
234 | """Update the file's path | |
217 |
|
235 | |||
@@ -431,3 +449,20 b' class ContentsManager(LoggingConfigurable):' | |||||
431 | def should_list(self, name): |
|
449 | def should_list(self, name): | |
432 | """Should this file/directory name be displayed in a listing?""" |
|
450 | """Should this file/directory name be displayed in a listing?""" | |
433 | return not any(fnmatch(name, glob) for glob in self.hide_globs) |
|
451 | return not any(fnmatch(name, glob) for glob in self.hide_globs) | |
|
452 | ||||
|
453 | # Part 3: Checkpoints API | |||
|
454 | def create_checkpoint(self, path): | |||
|
455 | """Create a checkpoint.""" | |||
|
456 | return self.checkpoints.create_checkpoint(self, path) | |||
|
457 | ||||
|
458 | def restore_checkpoint(self, checkpoint_id, path): | |||
|
459 | """ | |||
|
460 | Restore a checkpoint. | |||
|
461 | """ | |||
|
462 | self.checkpoints.restore_checkpoint(self, checkpoint_id, path) | |||
|
463 | ||||
|
464 | def list_checkpoints(self, path): | |||
|
465 | return self.checkpoints.list_checkpoints(path) | |||
|
466 | ||||
|
467 | def delete_checkpoint(self, checkpoint_id, path): | |||
|
468 | return self.checkpoints.delete_checkpoint(checkpoint_id, path) |
@@ -2,6 +2,7 b'' | |||||
2 | """Test the contents webservice API.""" |
|
2 | """Test the contents webservice API.""" | |
3 |
|
3 | |||
4 | import base64 |
|
4 | import base64 | |
|
5 | from contextlib import contextmanager | |||
5 | import io |
|
6 | import io | |
6 | import json |
|
7 | import json | |
7 | import os |
|
8 | import os | |
@@ -12,6 +13,9 b' pjoin = os.path.join' | |||||
12 |
|
13 | |||
13 | import requests |
|
14 | import requests | |
14 |
|
15 | |||
|
16 | from ..filecheckpoints import GenericFileCheckpoints | |||
|
17 | ||||
|
18 | from IPython.config import Config | |||
15 | from IPython.html.utils import url_path_join, url_escape, to_os_path |
|
19 | from IPython.html.utils import url_path_join, url_escape, to_os_path | |
16 | from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error |
|
20 | from IPython.html.tests.launchnotebook import NotebookTestBase, assert_http_error | |
17 | from IPython.nbformat import read, write, from_dict |
|
21 | from IPython.nbformat import read, write, from_dict | |
@@ -21,6 +25,7 b' from IPython.nbformat.v4 import (' | |||||
21 | from IPython.nbformat import v2 |
|
25 | from IPython.nbformat import v2 | |
22 | from IPython.utils import py3compat |
|
26 | from IPython.utils import py3compat | |
23 | from IPython.utils.data import uniq_stable |
|
27 | from IPython.utils.data import uniq_stable | |
|
28 | from IPython.utils.tempdir import TemporaryDirectory | |||
24 |
|
29 | |||
25 |
|
30 | |||
26 | def notebooks_only(dir_model): |
|
31 | def notebooks_only(dir_model): | |
@@ -502,7 +507,6 b' class APITest(NotebookTestBase):' | |||||
502 | self.assertEqual(newnb.cells[0].source, |
|
507 | self.assertEqual(newnb.cells[0].source, | |
503 | u'Created by test ³') |
|
508 | u'Created by test ³') | |
504 |
|
509 | |||
505 |
|
||||
506 | def test_checkpoints(self): |
|
510 | def test_checkpoints(self): | |
507 | resp = self.api.read('foo/a.ipynb') |
|
511 | resp = self.api.read('foo/a.ipynb') | |
508 | r = self.api.new_checkpoint('foo/a.ipynb') |
|
512 | r = self.api.new_checkpoint('foo/a.ipynb') | |
@@ -540,3 +544,93 b' class APITest(NotebookTestBase):' | |||||
540 | self.assertEqual(r.status_code, 204) |
|
544 | self.assertEqual(r.status_code, 204) | |
541 | cps = self.api.get_checkpoints('foo/a.ipynb').json() |
|
545 | cps = self.api.get_checkpoints('foo/a.ipynb').json() | |
542 | self.assertEqual(cps, []) |
|
546 | self.assertEqual(cps, []) | |
|
547 | ||||
|
548 | def test_file_checkpoints(self): | |||
|
549 | """ | |||
|
550 | Test checkpointing of non-notebook files. | |||
|
551 | """ | |||
|
552 | filename = 'foo/a.txt' | |||
|
553 | resp = self.api.read(filename) | |||
|
554 | orig_content = json.loads(resp.text)['content'] | |||
|
555 | ||||
|
556 | # Create a checkpoint. | |||
|
557 | r = self.api.new_checkpoint(filename) | |||
|
558 | self.assertEqual(r.status_code, 201) | |||
|
559 | cp1 = r.json() | |||
|
560 | self.assertEqual(set(cp1), {'id', 'last_modified'}) | |||
|
561 | self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id']) | |||
|
562 | ||||
|
563 | # Modify the file and save. | |||
|
564 | new_content = orig_content + '\nsecond line' | |||
|
565 | model = { | |||
|
566 | 'content': new_content, | |||
|
567 | 'type': 'file', | |||
|
568 | 'format': 'text', | |||
|
569 | } | |||
|
570 | resp = self.api.save(filename, body=json.dumps(model)) | |||
|
571 | ||||
|
572 | # List checkpoints | |||
|
573 | cps = self.api.get_checkpoints(filename).json() | |||
|
574 | self.assertEqual(cps, [cp1]) | |||
|
575 | ||||
|
576 | content = self.api.read(filename).json()['content'] | |||
|
577 | self.assertEqual(content, new_content) | |||
|
578 | ||||
|
579 | # Restore cp1 | |||
|
580 | r = self.api.restore_checkpoint(filename, cp1['id']) | |||
|
581 | self.assertEqual(r.status_code, 204) | |||
|
582 | restored_content = self.api.read(filename).json()['content'] | |||
|
583 | self.assertEqual(restored_content, orig_content) | |||
|
584 | ||||
|
585 | # Delete cp1 | |||
|
586 | r = self.api.delete_checkpoint(filename, cp1['id']) | |||
|
587 | self.assertEqual(r.status_code, 204) | |||
|
588 | cps = self.api.get_checkpoints(filename).json() | |||
|
589 | self.assertEqual(cps, []) | |||
|
590 | ||||
|
591 | @contextmanager | |||
|
592 | def patch_cp_root(self, dirname): | |||
|
593 | """ | |||
|
594 | Temporarily patch the root dir of our checkpoint manager. | |||
|
595 | """ | |||
|
596 | cpm = self.notebook.contents_manager.checkpoints | |||
|
597 | old_dirname = cpm.root_dir | |||
|
598 | cpm.root_dir = dirname | |||
|
599 | try: | |||
|
600 | yield | |||
|
601 | finally: | |||
|
602 | cpm.root_dir = old_dirname | |||
|
603 | ||||
|
604 | def test_checkpoints_separate_root(self): | |||
|
605 | """ | |||
|
606 | Test that FileCheckpoints functions correctly even when it's | |||
|
607 | using a different root dir from FileContentsManager. This also keeps | |||
|
608 | the implementation honest for use with ContentsManagers that don't map | |||
|
609 | models to the filesystem | |||
|
610 | ||||
|
611 | Override this method to a no-op when testing other managers. | |||
|
612 | """ | |||
|
613 | with TemporaryDirectory() as td: | |||
|
614 | with self.patch_cp_root(td): | |||
|
615 | self.test_checkpoints() | |||
|
616 | ||||
|
617 | with TemporaryDirectory() as td: | |||
|
618 | with self.patch_cp_root(td): | |||
|
619 | self.test_file_checkpoints() | |||
|
620 | ||||
|
621 | ||||
|
622 | class GenericFileCheckpointsAPITest(APITest): | |||
|
623 | """ | |||
|
624 | Run the tests from APITest with GenericFileCheckpoints. | |||
|
625 | """ | |||
|
626 | config = Config() | |||
|
627 | config.FileContentsManager.checkpoints_class = GenericFileCheckpoints | |||
|
628 | ||||
|
629 | def test_config_did_something(self): | |||
|
630 | ||||
|
631 | self.assertIsInstance( | |||
|
632 | self.notebook.contents_manager.checkpoints, | |||
|
633 | GenericFileCheckpoints, | |||
|
634 | ) | |||
|
635 | ||||
|
636 |
@@ -84,11 +84,16 b' class TestFileContentsManager(TestCase):' | |||||
84 | root = td |
|
84 | root = td | |
85 | os.mkdir(os.path.join(td, subd)) |
|
85 | os.mkdir(os.path.join(td, subd)) | |
86 | fm = FileContentsManager(root_dir=root) |
|
86 | fm = FileContentsManager(root_dir=root) | |
87 |
cp |
|
87 | cpm = fm.checkpoints | |
88 |
cp_ |
|
88 | cp_dir = cpm.checkpoint_path( | |
|
89 | 'cp', 'test.ipynb' | |||
|
90 | ) | |||
|
91 | cp_subdir = cpm.checkpoint_path( | |||
|
92 | 'cp', '/%s/test.ipynb' % subd | |||
|
93 | ) | |||
89 | self.assertNotEqual(cp_dir, cp_subdir) |
|
94 | self.assertNotEqual(cp_dir, cp_subdir) | |
90 |
self.assertEqual(cp_dir, os.path.join(root, |
|
95 | self.assertEqual(cp_dir, os.path.join(root, cpm.checkpoint_dir, cp_name)) | |
91 |
self.assertEqual(cp_subdir, os.path.join(root, subd, |
|
96 | self.assertEqual(cp_subdir, os.path.join(root, subd, cpm.checkpoint_dir, cp_name)) | |
92 |
|
97 | |||
93 | @dec.skip_win32 |
|
98 | @dec.skip_win32 | |
94 | def test_bad_symlink(self): |
|
99 | def test_bad_symlink(self): |
General Comments 0
You need to be logged in to leave comments.
Login now