Show More
@@ -60,7 +60,10 b' class FileManagerMixin(object):' | |||
|
60 | 60 | """ |
|
61 | 61 | Mixin for ContentsAPI classes that interact with the filesystem. |
|
62 | 62 | |
|
63 | Shared by both FileContentsManager and FileCheckpointManager. | |
|
63 | Provides facilities for reading, writing, and copying both notebooks and | |
|
64 | generic files. | |
|
65 | ||
|
66 | Shared by FileContentsManager and FileCheckpointManager. | |
|
64 | 67 | |
|
65 | 68 | Note |
|
66 | 69 | ---- |
@@ -114,17 +117,6 b' class FileManagerMixin(object):' | |||
|
114 | 117 | except OSError: |
|
115 | 118 | self.log.debug("copystat on %s failed", dest, exc_info=True) |
|
116 | 119 | |
|
117 | def _read_notebook(self, os_path, as_version=4): | |
|
118 | """Read a notebook from an os path.""" | |
|
119 | with self.open(os_path, 'r', encoding='utf-8') as f: | |
|
120 | try: | |
|
121 | return nbformat.read(f, as_version=as_version) | |
|
122 | except Exception as e: | |
|
123 | raise web.HTTPError( | |
|
124 | 400, | |
|
125 | u"Unreadable Notebook: %s %r" % (os_path, e), | |
|
126 | ) | |
|
127 | ||
|
128 | 120 | def _get_os_path(self, path): |
|
129 | 121 | """Given an API path, return its file system path. |
|
130 | 122 | |
@@ -140,6 +132,70 b' class FileManagerMixin(object):' | |||
|
140 | 132 | """ |
|
141 | 133 | return to_os_path(path, self.root_dir) |
|
142 | 134 | |
|
135 | def _read_notebook(self, os_path, as_version=4): | |
|
136 | """Read a notebook from an os path.""" | |
|
137 | with self.open(os_path, 'r', encoding='utf-8') as f: | |
|
138 | try: | |
|
139 | return nbformat.read(f, as_version=as_version) | |
|
140 | except Exception as e: | |
|
141 | raise web.HTTPError( | |
|
142 | 400, | |
|
143 | u"Unreadable Notebook: %s %r" % (os_path, e), | |
|
144 | ) | |
|
145 | ||
|
146 | def _save_notebook(self, os_path, nb): | |
|
147 | """Save a notebook to an os_path.""" | |
|
148 | with self.atomic_writing(os_path, encoding='utf-8') as f: | |
|
149 | nbformat.write(nb, f, version=nbformat.NO_CONVERT) | |
|
150 | ||
|
151 | def _read_file(self, os_path, format): | |
|
152 | """Read a non-notebook file. | |
|
153 | ||
|
154 | os_path: The path to be read. | |
|
155 | format: | |
|
156 | If 'text', the contents will be decoded as UTF-8. | |
|
157 | If 'base64', the raw bytes contents will be encoded as base64. | |
|
158 | If not specified, try to decode as UTF-8, and fall back to base64 | |
|
159 | """ | |
|
160 | if not os.path.isfile(os_path): | |
|
161 | raise web.HTTPError(400, "Cannot read non-file %s" % os_path) | |
|
162 | ||
|
163 | with self.open(os_path, 'rb') as f: | |
|
164 | bcontent = f.read() | |
|
165 | ||
|
166 | if format is None or format == 'text': | |
|
167 | # Try to interpret as unicode if format is unknown or if unicode | |
|
168 | # was explicitly requested. | |
|
169 | try: | |
|
170 | return bcontent.decode('utf8'), 'text' | |
|
171 | except UnicodeError as e: | |
|
172 | if format == 'text': | |
|
173 | raise web.HTTPError( | |
|
174 | 400, | |
|
175 | "%s is not UTF-8 encoded" % os_path, | |
|
176 | reason='bad format', | |
|
177 | ) | |
|
178 | return base64.encodestring(bcontent).decode('ascii'), 'base64' | |
|
179 | ||
|
180 | def _save_file(self, os_path, content, format): | |
|
181 | """Save content of a generic file.""" | |
|
182 | if format not in {'text', 'base64'}: | |
|
183 | raise web.HTTPError( | |
|
184 | 400, | |
|
185 | "Must specify format of file contents as 'text' or 'base64'", | |
|
186 | ) | |
|
187 | try: | |
|
188 | if format == 'text': | |
|
189 | bcontent = content.encode('utf8') | |
|
190 | else: | |
|
191 | b64_bytes = content.encode('ascii') | |
|
192 | bcontent = base64.decodestring(b64_bytes) | |
|
193 | except Exception as e: | |
|
194 | raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e)) | |
|
195 | ||
|
196 | with self.atomic_writing(os_path, text=False) as f: | |
|
197 | f.write(bcontent) | |
|
198 | ||
|
143 | 199 | |
|
144 | 200 | class FileCheckpointManager(FileManagerMixin, CheckpointManager): |
|
145 | 201 | """ |
@@ -167,39 +223,51 b' class FileCheckpointManager(FileManagerMixin, CheckpointManager):' | |||
|
167 | 223 | return getcwd() |
|
168 | 224 | |
|
169 | 225 | # public checkpoint API |
|
170 |
def create_checkpoint(self, |
|
|
226 | def create_file_checkpoint(self, content, format, path): | |
|
171 | 227 | """Create a checkpoint from the current content of a notebook.""" |
|
172 | 228 | path = path.strip('/') |
|
173 | 229 | # only the one checkpoint ID: |
|
174 | 230 | checkpoint_id = u"checkpoint" |
|
175 |
os_checkpoint_path = self. |
|
|
231 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |
|
232 | self.log.debug("creating checkpoint for %s", path) | |
|
233 | with self.perm_to_403(): | |
|
234 | self._save_file(os_checkpoint_path, content, format=format) | |
|
235 | ||
|
236 | # return the checkpoint info | |
|
237 | return self.checkpoint_model(checkpoint_id, os_checkpoint_path) | |
|
238 | ||
|
239 | def create_notebook_checkpoint(self, nb, path): | |
|
240 | """Create a checkpoint from the current content of a notebook.""" | |
|
241 | path = path.strip('/') | |
|
242 | # only the one checkpoint ID: | |
|
243 | checkpoint_id = u"checkpoint" | |
|
244 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |
|
176 | 245 | self.log.debug("creating checkpoint for %s", path) |
|
177 | 246 | with self.perm_to_403(): |
|
178 | 247 | self._save_notebook(os_checkpoint_path, nb) |
|
179 | 248 | |
|
180 | 249 | # return the checkpoint info |
|
181 |
return self. |
|
|
250 | return self.checkpoint_model(checkpoint_id, os_checkpoint_path) | |
|
182 | 251 | |
|
183 |
def get_checkpoint |
|
|
252 | def get_checkpoint(self, checkpoint_id, path, type): | |
|
184 | 253 | """Get the content of a checkpoint. |
|
185 | 254 | |
|
186 | Returns an unvalidated model with the same structure as | |
|
187 | the return value of ContentsManager.get | |
|
255 | Returns a pair of (content, type). | |
|
188 | 256 | """ |
|
189 | 257 | path = path.strip('/') |
|
190 | 258 | self.log.info("restoring %s from checkpoint %s", path, checkpoint_id) |
|
191 |
os_checkpoint_path = self. |
|
|
192 | return self._read_notebook(os_checkpoint_path, as_version=4) | |
|
193 | ||
|
194 | def _save_notebook(self, os_path, nb): | |
|
195 | """Save a notebook file.""" | |
|
196 | with self.atomic_writing(os_path, encoding='utf-8') as f: | |
|
197 | nbformat.write(nb, f, version=nbformat.NO_CONVERT) | |
|
259 | os_checkpoint_path = self.checkpoint_path(checkpoint_id, path) | |
|
260 | if not os.path.isfile(os_checkpoint_path): | |
|
261 | self.no_such_checkpoint(path, checkpoint_id) | |
|
262 | if type == 'notebook': | |
|
263 | return self._read_notebook(os_checkpoint_path, as_version=4), None | |
|
264 | else: | |
|
265 | return self._read_file(os_checkpoint_path, format=None) | |
|
198 | 266 | |
|
199 | 267 | def rename_checkpoint(self, checkpoint_id, old_path, new_path): |
|
200 | 268 | """Rename a checkpoint from old_path to new_path.""" |
|
201 |
old_cp_path = self. |
|
|
202 |
new_cp_path = self. |
|
|
269 | old_cp_path = self.checkpoint_path(checkpoint_id, old_path) | |
|
270 | new_cp_path = self.checkpoint_path(checkpoint_id, new_path) | |
|
203 | 271 | if os.path.isfile(old_cp_path): |
|
204 | 272 | self.log.debug( |
|
205 | 273 | "Renaming checkpoint %s -> %s", |
@@ -212,7 +280,7 b' class FileCheckpointManager(FileManagerMixin, CheckpointManager):' | |||
|
212 | 280 | def delete_checkpoint(self, checkpoint_id, path): |
|
213 | 281 | """delete a file's checkpoint""" |
|
214 | 282 | path = path.strip('/') |
|
215 |
cp_path = self. |
|
|
283 | cp_path = self.checkpoint_path(checkpoint_id, path) | |
|
216 | 284 | if not os.path.isfile(cp_path): |
|
217 | 285 | self.no_such_checkpoint(path, checkpoint_id) |
|
218 | 286 | |
@@ -227,14 +295,14 b' class FileCheckpointManager(FileManagerMixin, CheckpointManager):' | |||
|
227 | 295 | """ |
|
228 | 296 | path = path.strip('/') |
|
229 | 297 | checkpoint_id = "checkpoint" |
|
230 |
os_path = self. |
|
|
231 |
if not os.path. |
|
|
298 | os_path = self.checkpoint_path(checkpoint_id, path) | |
|
299 | if not os.path.isfile(os_path): | |
|
232 | 300 | return [] |
|
233 | 301 | else: |
|
234 |
return [self. |
|
|
302 | return [self.checkpoint_model(checkpoint_id, os_path)] | |
|
235 | 303 | |
|
236 | 304 | # Checkpoint-related utilities |
|
237 |
def |
|
|
305 | def checkpoint_path(self, checkpoint_id, path): | |
|
238 | 306 | """find the path to a checkpoint""" |
|
239 | 307 | path = path.strip('/') |
|
240 | 308 | parent, name = ('/' + path).rsplit('/', 1) |
@@ -252,11 +320,9 b' class FileCheckpointManager(FileManagerMixin, CheckpointManager):' | |||
|
252 | 320 | cp_path = os.path.join(cp_dir, filename) |
|
253 | 321 | return cp_path |
|
254 | 322 | |
|
255 |
def |
|
|
323 | def checkpoint_model(self, checkpoint_id, os_path): | |
|
256 | 324 | """construct the info dict for a given checkpoint""" |
|
257 | path = path.strip('/') | |
|
258 | cp_path = self.get_checkpoint_path(checkpoint_id, path) | |
|
259 | stats = os.stat(cp_path) | |
|
325 | stats = os.stat(os_path) | |
|
260 | 326 | last_modified = tz.utcfromtimestamp(stats.st_mtime) |
|
261 | 327 | info = dict( |
|
262 | 328 | id=checkpoint_id, |
@@ -499,29 +565,17 b' class FileContentsManager(FileManagerMixin, ContentsManager):' | |||
|
499 | 565 | os_path = self._get_os_path(path) |
|
500 | 566 | |
|
501 | 567 | if content: |
|
502 |
|
|
|
503 | # could be FIFO | |
|
504 | raise web.HTTPError(400, "Cannot get content of non-file %s" % os_path) | |
|
505 | with self.open(os_path, 'rb') as f: | |
|
506 | bcontent = f.read() | |
|
507 | ||
|
508 | if format != 'base64': | |
|
509 |
|
|
|
510 | model['content'] = bcontent.decode('utf8') | |
|
511 | except UnicodeError as e: | |
|
512 | if format == 'text': | |
|
513 | raise web.HTTPError(400, "%s is not UTF-8 encoded" % path, reason='bad format') | |
|
514 | else: | |
|
515 | model['format'] = 'text' | |
|
516 | default_mime = 'text/plain' | |
|
517 | ||
|
518 | if model['content'] is None: | |
|
519 | model['content'] = base64.encodestring(bcontent).decode('ascii') | |
|
520 | model['format'] = 'base64' | |
|
521 | if model['format'] == 'base64': | |
|
522 | default_mime = 'application/octet-stream' | |
|
523 | ||
|
524 | model['mimetype'] = mimetypes.guess_type(os_path)[0] or default_mime | |
|
568 | content, format = self._read_file(os_path, format) | |
|
569 | default_mime = { | |
|
570 | 'text': 'text/plain', | |
|
571 | 'base64': 'application/octet-stream' | |
|
572 | }[format] | |
|
573 | ||
|
574 | model.update( | |
|
575 | content=content, | |
|
576 | format=format, | |
|
577 | mimetype=mimetypes.guess_type(os_path)[0] or default_mime, | |
|
578 | ) | |
|
525 | 579 | |
|
526 | 580 | return model |
|
527 | 581 | |
@@ -584,35 +638,6 b' class FileContentsManager(FileManagerMixin, ContentsManager):' | |||
|
584 | 638 | model = self._file_model(path, content=content, format=format) |
|
585 | 639 | return model |
|
586 | 640 | |
|
587 | def _save_notebook(self, os_path, model, path): | |
|
588 | """save a notebook file""" | |
|
589 | nb = nbformat.from_dict(model['content']) | |
|
590 | self.check_and_sign(nb, path) | |
|
591 | ||
|
592 | # One checkpoint should always exist for notebooks. | |
|
593 | if not self.checkpoint_manager.list_checkpoints(path): | |
|
594 | self.checkpoint_manager.create_checkpoint(nb, path) | |
|
595 | ||
|
596 | with self.atomic_writing(os_path, encoding='utf-8') as f: | |
|
597 | nbformat.write(nb, f, version=nbformat.NO_CONVERT) | |
|
598 | ||
|
599 | def _save_file(self, os_path, model, path=''): | |
|
600 | """save a non-notebook file""" | |
|
601 | fmt = model.get('format', None) | |
|
602 | if fmt not in {'text', 'base64'}: | |
|
603 | raise web.HTTPError(400, "Must specify format of file contents as 'text' or 'base64'") | |
|
604 | try: | |
|
605 | content = model['content'] | |
|
606 | if fmt == 'text': | |
|
607 | bcontent = content.encode('utf8') | |
|
608 | else: | |
|
609 | b64_bytes = content.encode('ascii') | |
|
610 | bcontent = base64.decodestring(b64_bytes) | |
|
611 | except Exception as e: | |
|
612 | raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e)) | |
|
613 | with self.atomic_writing(os_path, text=False) as f: | |
|
614 | f.write(bcontent) | |
|
615 | ||
|
616 | 641 | def _save_directory(self, os_path, model, path=''): |
|
617 | 642 | """create a directory""" |
|
618 | 643 | if is_hidden(os_path, self.root_dir): |
@@ -640,9 +665,18 b' class FileContentsManager(FileManagerMixin, ContentsManager):' | |||
|
640 | 665 | self.log.debug("Saving %s", os_path) |
|
641 | 666 | try: |
|
642 | 667 | if model['type'] == 'notebook': |
|
643 | self._save_notebook(os_path, model, path) | |
|
668 | nb = nbformat.from_dict(model['content']) | |
|
669 | self.check_and_sign(nb, path) | |
|
670 | self._save_notebook(os_path, nb) | |
|
671 | # One checkpoint should always exist for notebooks. | |
|
672 | if not self.checkpoint_manager.list_checkpoints(path): | |
|
673 | self.checkpoint_manager.create_notebook_checkpoint( | |
|
674 | nb, | |
|
675 | path, | |
|
676 | ) | |
|
644 | 677 | elif model['type'] == 'file': |
|
645 | self._save_file(os_path, model, path) | |
|
678 | # Missing format will be handled internally by _save_file. | |
|
679 | self._save_file(os_path, model['content'], model.get('format')) | |
|
646 | 680 | elif model['type'] == 'directory': |
|
647 | 681 | self._save_directory(os_path, model, path) |
|
648 | 682 | else: |
@@ -34,15 +34,21 b' class CheckpointManager(LoggingConfigurable):' | |||
|
34 | 34 | """ |
|
35 | 35 | Base class for managing checkpoints for a ContentsManager. |
|
36 | 36 | """ |
|
37 | def create_file_checkpoint(self, content, format, path): | |
|
38 | """Create a checkpoint of the current state of a file | |
|
39 | ||
|
40 | Returns a checkpoint model for the new checkpoint. | |
|
41 | """ | |
|
42 | raise NotImplementedError("must be implemented in a subclass") | |
|
37 | 43 | |
|
38 | def create_checkpoint(self, nb, path): | |
|
44 | def create_notebook_checkpoint(self, nb, path): | |
|
39 | 45 | """Create a checkpoint of the current state of a file |
|
40 | 46 | |
|
41 |
Returns a checkpoint |
|
|
47 | Returns a checkpoint model for the new checkpoint. | |
|
42 | 48 | """ |
|
43 | 49 | raise NotImplementedError("must be implemented in a subclass") |
|
44 | 50 | |
|
45 |
def get_checkpoint |
|
|
51 | def get_checkpoint(self, checkpoint_id, path, type): | |
|
46 | 52 | """Get the content of a checkpoint. |
|
47 | 53 | |
|
48 | 54 | Returns an unvalidated model with the same structure as |
@@ -496,9 +502,19 b' class ContentsManager(LoggingConfigurable):' | |||
|
496 | 502 | # Part 3: Checkpoints API |
|
497 | 503 | def create_checkpoint(self, path): |
|
498 | 504 | """Create a checkpoint.""" |
|
499 | ||
|
500 | nb = nbformat.from_dict(self.get(path, content=True)['content']) | |
|
501 | return self.checkpoint_manager.create_checkpoint(nb, path) | |
|
505 | model = self.get(path, content=True) | |
|
506 | type = model['type'] | |
|
507 | if type == 'notebook': | |
|
508 | return self.checkpoint_manager.create_notebook_checkpoint( | |
|
509 | model['content'], | |
|
510 | path, | |
|
511 | ) | |
|
512 | elif type == 'file': | |
|
513 | return self.checkpoint_manager.create_file_checkpoint( | |
|
514 | model['content'], | |
|
515 | model['format'], | |
|
516 | path, | |
|
517 | ) | |
|
502 | 518 | |
|
503 | 519 | def list_checkpoints(self, path): |
|
504 | 520 | return self.checkpoint_manager.list_checkpoints(path) |
@@ -507,17 +523,18 b' class ContentsManager(LoggingConfigurable):' | |||
|
507 | 523 | """ |
|
508 | 524 | Restore a checkpoint. |
|
509 | 525 | """ |
|
510 | nb = self.checkpoint_manager.get_checkpoint_content( | |
|
526 | type = self.get(path, content=False)['type'] | |
|
527 | content, format = self.checkpoint_manager.get_checkpoint( | |
|
511 | 528 | checkpoint_id, |
|
512 | 529 | path, |
|
530 | type, | |
|
513 | 531 | ) |
|
514 | 532 | |
|
515 | 533 | model = { |
|
516 |
' |
|
|
517 |
't |
|
|
534 | 'type': type, | |
|
535 | 'content': content, | |
|
536 | 'format': format, | |
|
518 | 537 | } |
|
519 | ||
|
520 | self.validate_notebook_model(model) | |
|
521 | 538 | return self.save(model, path) |
|
522 | 539 | |
|
523 | 540 | def delete_checkpoint(self, checkpoint_id, path): |
@@ -542,6 +542,49 b' class APITest(NotebookTestBase):' | |||
|
542 | 542 | cps = self.api.get_checkpoints('foo/a.ipynb').json() |
|
543 | 543 | self.assertEqual(cps, []) |
|
544 | 544 | |
|
545 | def test_file_checkpoints(self): | |
|
546 | """ | |
|
547 | Test checkpointing of non-notebook files. | |
|
548 | """ | |
|
549 | filename = 'foo/a.txt' | |
|
550 | resp = self.api.read(filename) | |
|
551 | orig_content = json.loads(resp.text)['content'] | |
|
552 | ||
|
553 | # Create a checkpoint. | |
|
554 | r = self.api.new_checkpoint(filename) | |
|
555 | self.assertEqual(r.status_code, 201) | |
|
556 | cp1 = r.json() | |
|
557 | self.assertEqual(set(cp1), {'id', 'last_modified'}) | |
|
558 | self.assertEqual(r.headers['Location'].split('/')[-1], cp1['id']) | |
|
559 | ||
|
560 | # Modify the file and save. | |
|
561 | new_content = orig_content + '\nsecond line' | |
|
562 | model = { | |
|
563 | 'content': new_content, | |
|
564 | 'type': 'file', | |
|
565 | 'format': 'text', | |
|
566 | } | |
|
567 | resp = self.api.save(filename, body=json.dumps(model)) | |
|
568 | ||
|
569 | # List checkpoints | |
|
570 | cps = self.api.get_checkpoints(filename).json() | |
|
571 | self.assertEqual(cps, [cp1]) | |
|
572 | ||
|
573 | content = self.api.read(filename).json()['content'] | |
|
574 | self.assertEqual(content, new_content) | |
|
575 | ||
|
576 | # Restore cp1 | |
|
577 | r = self.api.restore_checkpoint(filename, cp1['id']) | |
|
578 | self.assertEqual(r.status_code, 204) | |
|
579 | restored_content = self.api.read(filename).json()['content'] | |
|
580 | self.assertEqual(restored_content, orig_content) | |
|
581 | ||
|
582 | # Delete cp1 | |
|
583 | r = self.api.delete_checkpoint(filename, cp1['id']) | |
|
584 | self.assertEqual(r.status_code, 204) | |
|
585 | cps = self.api.get_checkpoints(filename).json() | |
|
586 | self.assertEqual(cps, []) | |
|
587 | ||
|
545 | 588 | @contextmanager |
|
546 | 589 | def patch_cp_root(self, dirname): |
|
547 | 590 | """ |
@@ -561,8 +604,13 b' class APITest(NotebookTestBase):' | |||
|
561 | 604 | using a different root dir from FileContentsManager. This also keeps |
|
562 | 605 | the implementation honest for use with ContentsManagers that don't map |
|
563 | 606 | models to the filesystem |
|
564 | """ | |
|
565 | 607 |
|
|
608 | Override this method to a no-op when testing other managers. | |
|
609 | """ | |
|
566 | 610 | with TemporaryDirectory() as td: |
|
567 | 611 | with self.patch_cp_root(td): |
|
568 | 612 | self.test_checkpoints() |
|
613 | ||
|
614 | with TemporaryDirectory() as td: | |
|
615 | with self.patch_cp_root(td): | |
|
616 | self.test_file_checkpoints() |
@@ -85,10 +85,10 b' class TestFileContentsManager(TestCase):' | |||
|
85 | 85 | os.mkdir(os.path.join(td, subd)) |
|
86 | 86 | fm = FileContentsManager(root_dir=root) |
|
87 | 87 | cpm = fm.checkpoint_manager |
|
88 |
cp_dir = cpm. |
|
|
88 | cp_dir = cpm.checkpoint_path( | |
|
89 | 89 | 'cp', 'test.ipynb' |
|
90 | 90 | ) |
|
91 |
cp_subdir = cpm. |
|
|
91 | cp_subdir = cpm.checkpoint_path( | |
|
92 | 92 | 'cp', '/%s/test.ipynb' % subd |
|
93 | 93 | ) |
|
94 | 94 | self.assertNotEqual(cp_dir, cp_subdir) |
General Comments 0
You need to be logged in to leave comments.
Login now