##// END OF EJS Templates
Merge pull request #7312 from quantopian/refactor-contents-update...
Min RK -
r19714:649baa24 merge
parent child Browse files
Show More
@@ -1,696 +1,683 b''
1 1 """A contents manager that uses the local file system for storage."""
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6 import base64
7 7 import errno
8 8 import io
9 9 import os
10 10 import shutil
11 11 from contextlib import contextmanager
12 12 import mimetypes
13 13
14 14 from tornado import web
15 15
16 16 from .manager import ContentsManager
17 17 from IPython import nbformat
18 18 from IPython.utils.io import atomic_writing
19 19 from IPython.utils.importstring import import_item
20 20 from IPython.utils.path import ensure_dir_exists
21 21 from IPython.utils.traitlets import Any, Unicode, Bool, TraitError
22 22 from IPython.utils.py3compat import getcwd, str_to_unicode, string_types
23 23 from IPython.utils import tz
24 24 from IPython.html.utils import is_hidden, to_os_path, to_api_path
25 25
26 26 _script_exporter = None
27 27
28 28 def _post_save_script(model, os_path, contents_manager, **kwargs):
29 29 """convert notebooks to Python script after save with nbconvert
30 30
31 31 replaces `ipython notebook --script`
32 32 """
33 33 from IPython.nbconvert.exporters.script import ScriptExporter
34 34
35 35 if model['type'] != 'notebook':
36 36 return
37 37
38 38 global _script_exporter
39 39 if _script_exporter is None:
40 40 _script_exporter = ScriptExporter(parent=contents_manager)
41 41 log = contents_manager.log
42 42
43 43 base, ext = os.path.splitext(os_path)
44 44 py_fname = base + '.py'
45 45 script, resources = _script_exporter.from_filename(os_path)
46 46 script_fname = base + resources.get('output_extension', '.txt')
47 47 log.info("Saving script /%s", to_api_path(script_fname, contents_manager.root_dir))
48 48 with io.open(script_fname, 'w', encoding='utf-8') as f:
49 49 f.write(script)
50 50
51 51 class FileContentsManager(ContentsManager):
52 52
53 53 root_dir = Unicode(config=True)
54 54
55 55 def _root_dir_default(self):
56 56 try:
57 57 return self.parent.notebook_dir
58 58 except AttributeError:
59 59 return getcwd()
60 60
61 61 @contextmanager
62 62 def perm_to_403(self, os_path=''):
63 63 """context manager for turning permission errors into 403"""
64 64 try:
65 65 yield
66 66 except OSError as e:
67 67 if e.errno in {errno.EPERM, errno.EACCES}:
68 68 # make 403 error message without root prefix
69 69 # this may not work perfectly on unicode paths on Python 2,
70 70 # but nobody should be doing that anyway.
71 71 if not os_path:
72 72 os_path = str_to_unicode(e.filename or 'unknown file')
73 73 path = to_api_path(os_path, self.root_dir)
74 74 raise web.HTTPError(403, u'Permission denied: %s' % path)
75 75 else:
76 76 raise
77 77
78 78 @contextmanager
79 79 def open(self, os_path, *args, **kwargs):
80 80 """wrapper around io.open that turns permission errors into 403"""
81 81 with self.perm_to_403(os_path):
82 82 with io.open(os_path, *args, **kwargs) as f:
83 83 yield f
84 84
85 85 @contextmanager
86 86 def atomic_writing(self, os_path, *args, **kwargs):
87 87 """wrapper around atomic_writing that turns permission errors into 403"""
88 88 with self.perm_to_403(os_path):
89 89 with atomic_writing(os_path, *args, **kwargs) as f:
90 90 yield f
91 91
92 92 save_script = Bool(False, config=True, help='DEPRECATED, use post_save_hook')
93 93 def _save_script_changed(self):
94 94 self.log.warn("""
95 95 `--script` is deprecated. You can trigger nbconvert via pre- or post-save hooks:
96 96
97 97 ContentsManager.pre_save_hook
98 98 FileContentsManager.post_save_hook
99 99
100 100 A post-save hook has been registered that calls:
101 101
102 102 ipython nbconvert --to script [notebook]
103 103
104 104 which behaves similarly to `--script`.
105 105 """)
106 106
107 107 self.post_save_hook = _post_save_script
108 108
109 109 post_save_hook = Any(None, config=True,
110 110 help="""Python callable or importstring thereof
111 111
112 112 to be called on the path of a file just saved.
113 113
114 114 This can be used to process the file on disk,
115 115 such as converting the notebook to a script or HTML via nbconvert.
116 116
117 117 It will be called as (all arguments passed by keyword):
118 118
119 119 hook(os_path=os_path, model=model, contents_manager=instance)
120 120
121 121 path: the filesystem path to the file just written
122 122 model: the model representing the file
123 123 contents_manager: this ContentsManager instance
124 124 """
125 125 )
126 126 def _post_save_hook_changed(self, name, old, new):
127 127 if new and isinstance(new, string_types):
128 128 self.post_save_hook = import_item(self.post_save_hook)
129 129 elif new:
130 130 if not callable(new):
131 131 raise TraitError("post_save_hook must be callable")
132 132
133 133 def run_post_save_hook(self, model, os_path):
134 134 """Run the post-save hook if defined, and log errors"""
135 135 if self.post_save_hook:
136 136 try:
137 137 self.log.debug("Running post-save hook on %s", os_path)
138 138 self.post_save_hook(os_path=os_path, model=model, contents_manager=self)
139 139 except Exception:
140 140 self.log.error("Post-save hook failed on %s", os_path, exc_info=True)
141 141
142 142 def _root_dir_changed(self, name, old, new):
143 143 """Do a bit of validation of the root_dir."""
144 144 if not os.path.isabs(new):
145 145 # If we receive a non-absolute path, make it absolute.
146 146 self.root_dir = os.path.abspath(new)
147 147 return
148 148 if not os.path.isdir(new):
149 149 raise TraitError("%r is not a directory" % new)
150 150
151 151 checkpoint_dir = Unicode('.ipynb_checkpoints', config=True,
152 152 help="""The directory name in which to keep file checkpoints
153 153
154 154 This is a path relative to the file's own directory.
155 155
156 156 By default, it is .ipynb_checkpoints
157 157 """
158 158 )
159 159
160 160 def _copy(self, src, dest):
161 161 """copy src to dest
162 162
163 163 like shutil.copy2, but log errors in copystat
164 164 """
165 165 shutil.copyfile(src, dest)
166 166 try:
167 167 shutil.copystat(src, dest)
168 168 except OSError as e:
169 169 self.log.debug("copystat on %s failed", dest, exc_info=True)
170 170
171 171 def _get_os_path(self, path):
172 172 """Given an API path, return its file system path.
173 173
174 174 Parameters
175 175 ----------
176 176 path : string
177 177 The relative API path to the named file.
178 178
179 179 Returns
180 180 -------
181 181 path : string
182 182 Native, absolute OS path to for a file.
183 183 """
184 184 return to_os_path(path, self.root_dir)
185 185
186 186 def dir_exists(self, path):
187 187 """Does the API-style path refer to an extant directory?
188 188
189 189 API-style wrapper for os.path.isdir
190 190
191 191 Parameters
192 192 ----------
193 193 path : string
194 194 The path to check. This is an API path (`/` separated,
195 195 relative to root_dir).
196 196
197 197 Returns
198 198 -------
199 199 exists : bool
200 200 Whether the path is indeed a directory.
201 201 """
202 202 path = path.strip('/')
203 203 os_path = self._get_os_path(path=path)
204 204 return os.path.isdir(os_path)
205 205
206 206 def is_hidden(self, path):
207 207 """Does the API style path correspond to a hidden directory or file?
208 208
209 209 Parameters
210 210 ----------
211 211 path : string
212 212 The path to check. This is an API path (`/` separated,
213 213 relative to root_dir).
214 214
215 215 Returns
216 216 -------
217 217 hidden : bool
218 218 Whether the path exists and is hidden.
219 219 """
220 220 path = path.strip('/')
221 221 os_path = self._get_os_path(path=path)
222 222 return is_hidden(os_path, self.root_dir)
223 223
224 224 def file_exists(self, path):
225 225 """Returns True if the file exists, else returns False.
226 226
227 227 API-style wrapper for os.path.isfile
228 228
229 229 Parameters
230 230 ----------
231 231 path : string
232 232 The relative path to the file (with '/' as separator)
233 233
234 234 Returns
235 235 -------
236 236 exists : bool
237 237 Whether the file exists.
238 238 """
239 239 path = path.strip('/')
240 240 os_path = self._get_os_path(path)
241 241 return os.path.isfile(os_path)
242 242
243 243 def exists(self, path):
244 244 """Returns True if the path exists, else returns False.
245 245
246 246 API-style wrapper for os.path.exists
247 247
248 248 Parameters
249 249 ----------
250 250 path : string
251 251 The API path to the file (with '/' as separator)
252 252
253 253 Returns
254 254 -------
255 255 exists : bool
256 256 Whether the target exists.
257 257 """
258 258 path = path.strip('/')
259 259 os_path = self._get_os_path(path=path)
260 260 return os.path.exists(os_path)
261 261
262 262 def _base_model(self, path):
263 263 """Build the common base of a contents model"""
264 264 os_path = self._get_os_path(path)
265 265 info = os.stat(os_path)
266 266 last_modified = tz.utcfromtimestamp(info.st_mtime)
267 267 created = tz.utcfromtimestamp(info.st_ctime)
268 268 # Create the base model.
269 269 model = {}
270 270 model['name'] = path.rsplit('/', 1)[-1]
271 271 model['path'] = path
272 272 model['last_modified'] = last_modified
273 273 model['created'] = created
274 274 model['content'] = None
275 275 model['format'] = None
276 276 model['mimetype'] = None
277 277 try:
278 278 model['writable'] = os.access(os_path, os.W_OK)
279 279 except OSError:
280 280 self.log.error("Failed to check write permissions on %s", os_path)
281 281 model['writable'] = False
282 282 return model
283 283
284 284 def _dir_model(self, path, content=True):
285 285 """Build a model for a directory
286 286
287 287 if content is requested, will include a listing of the directory
288 288 """
289 289 os_path = self._get_os_path(path)
290 290
291 291 four_o_four = u'directory does not exist: %r' % path
292 292
293 293 if not os.path.isdir(os_path):
294 294 raise web.HTTPError(404, four_o_four)
295 295 elif is_hidden(os_path, self.root_dir):
296 296 self.log.info("Refusing to serve hidden directory %r, via 404 Error",
297 297 os_path
298 298 )
299 299 raise web.HTTPError(404, four_o_four)
300 300
301 301 model = self._base_model(path)
302 302 model['type'] = 'directory'
303 303 if content:
304 304 model['content'] = contents = []
305 305 os_dir = self._get_os_path(path)
306 306 for name in os.listdir(os_dir):
307 307 os_path = os.path.join(os_dir, name)
308 308 # skip over broken symlinks in listing
309 309 if not os.path.exists(os_path):
310 310 self.log.warn("%s doesn't exist", os_path)
311 311 continue
312 312 elif not os.path.isfile(os_path) and not os.path.isdir(os_path):
313 313 self.log.debug("%s not a regular file", os_path)
314 314 continue
315 315 if self.should_list(name) and not is_hidden(os_path, self.root_dir):
316 316 contents.append(self.get(
317 317 path='%s/%s' % (path, name),
318 318 content=False)
319 319 )
320 320
321 321 model['format'] = 'json'
322 322
323 323 return model
324 324
325 325 def _file_model(self, path, content=True, format=None):
326 326 """Build a model for a file
327 327
328 328 if content is requested, include the file contents.
329 329
330 330 format:
331 331 If 'text', the contents will be decoded as UTF-8.
332 332 If 'base64', the raw bytes contents will be encoded as base64.
333 333 If not specified, try to decode as UTF-8, and fall back to base64
334 334 """
335 335 model = self._base_model(path)
336 336 model['type'] = 'file'
337 337
338 338 os_path = self._get_os_path(path)
339 339
340 340 if content:
341 341 if not os.path.isfile(os_path):
342 342 # could be FIFO
343 343 raise web.HTTPError(400, "Cannot get content of non-file %s" % os_path)
344 344 with self.open(os_path, 'rb') as f:
345 345 bcontent = f.read()
346 346
347 347 if format != 'base64':
348 348 try:
349 349 model['content'] = bcontent.decode('utf8')
350 350 except UnicodeError as e:
351 351 if format == 'text':
352 352 raise web.HTTPError(400, "%s is not UTF-8 encoded" % path, reason='bad format')
353 353 else:
354 354 model['format'] = 'text'
355 355 default_mime = 'text/plain'
356 356
357 357 if model['content'] is None:
358 358 model['content'] = base64.encodestring(bcontent).decode('ascii')
359 359 model['format'] = 'base64'
360 360 if model['format'] == 'base64':
361 361 default_mime = 'application/octet-stream'
362 362
363 363 model['mimetype'] = mimetypes.guess_type(os_path)[0] or default_mime
364 364
365 365 return model
366 366
367 367
368 368 def _notebook_model(self, path, content=True):
369 369 """Build a notebook model
370 370
371 371 if content is requested, the notebook content will be populated
372 372 as a JSON structure (not double-serialized)
373 373 """
374 374 model = self._base_model(path)
375 375 model['type'] = 'notebook'
376 376 if content:
377 377 os_path = self._get_os_path(path)
378 378 with self.open(os_path, 'r', encoding='utf-8') as f:
379 379 try:
380 380 nb = nbformat.read(f, as_version=4)
381 381 except Exception as e:
382 382 raise web.HTTPError(400, u"Unreadable Notebook: %s %r" % (os_path, e))
383 383 self.mark_trusted_cells(nb, path)
384 384 model['content'] = nb
385 385 model['format'] = 'json'
386 386 self.validate_notebook_model(model)
387 387 return model
388 388
389 389 def get(self, path, content=True, type=None, format=None):
390 390 """ Takes a path for an entity and returns its model
391 391
392 392 Parameters
393 393 ----------
394 394 path : str
395 395 the API path that describes the relative path for the target
396 396 content : bool
397 397 Whether to include the contents in the reply
398 398 type : str, optional
399 399 The requested type - 'file', 'notebook', or 'directory'.
400 400 Will raise HTTPError 400 if the content doesn't match.
401 401 format : str, optional
402 402 The requested format for file contents. 'text' or 'base64'.
403 403 Ignored if this returns a notebook or directory model.
404 404
405 405 Returns
406 406 -------
407 407 model : dict
408 408 the contents model. If content=True, returns the contents
409 409 of the file or directory as well.
410 410 """
411 411 path = path.strip('/')
412 412
413 413 if not self.exists(path):
414 414 raise web.HTTPError(404, u'No such file or directory: %s' % path)
415 415
416 416 os_path = self._get_os_path(path)
417 417 if os.path.isdir(os_path):
418 418 if type not in (None, 'directory'):
419 419 raise web.HTTPError(400,
420 420 u'%s is a directory, not a %s' % (path, type), reason='bad type')
421 421 model = self._dir_model(path, content=content)
422 422 elif type == 'notebook' or (type is None and path.endswith('.ipynb')):
423 423 model = self._notebook_model(path, content=content)
424 424 else:
425 425 if type == 'directory':
426 426 raise web.HTTPError(400,
427 427 u'%s is not a directory', reason='bad type')
428 428 model = self._file_model(path, content=content, format=format)
429 429 return model
430 430
431 431 def _save_notebook(self, os_path, model, path=''):
432 432 """save a notebook file"""
433 433 # Save the notebook file
434 434 nb = nbformat.from_dict(model['content'])
435 435
436 436 self.check_and_sign(nb, path)
437 437
438 438 with self.atomic_writing(os_path, encoding='utf-8') as f:
439 439 nbformat.write(nb, f, version=nbformat.NO_CONVERT)
440 440
441 441 def _save_file(self, os_path, model, path=''):
442 442 """save a non-notebook file"""
443 443 fmt = model.get('format', None)
444 444 if fmt not in {'text', 'base64'}:
445 445 raise web.HTTPError(400, "Must specify format of file contents as 'text' or 'base64'")
446 446 try:
447 447 content = model['content']
448 448 if fmt == 'text':
449 449 bcontent = content.encode('utf8')
450 450 else:
451 451 b64_bytes = content.encode('ascii')
452 452 bcontent = base64.decodestring(b64_bytes)
453 453 except Exception as e:
454 454 raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e))
455 455 with self.atomic_writing(os_path, text=False) as f:
456 456 f.write(bcontent)
457 457
458 458 def _save_directory(self, os_path, model, path=''):
459 459 """create a directory"""
460 460 if is_hidden(os_path, self.root_dir):
461 461 raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path)
462 462 if not os.path.exists(os_path):
463 463 with self.perm_to_403():
464 464 os.mkdir(os_path)
465 465 elif not os.path.isdir(os_path):
466 466 raise web.HTTPError(400, u'Not a directory: %s' % (os_path))
467 467 else:
468 468 self.log.debug("Directory %r already exists", os_path)
469 469
470 470 def save(self, model, path=''):
471 471 """Save the file model and return the model with no content."""
472 472 path = path.strip('/')
473 473
474 474 if 'type' not in model:
475 475 raise web.HTTPError(400, u'No file type provided')
476 476 if 'content' not in model and model['type'] != 'directory':
477 477 raise web.HTTPError(400, u'No file content provided')
478 478
479 479 self.run_pre_save_hook(model=model, path=path)
480 480
481 481 # One checkpoint should always exist
482 482 if self.file_exists(path) and not self.list_checkpoints(path):
483 483 self.create_checkpoint(path)
484 484
485 485 os_path = self._get_os_path(path)
486 486 self.log.debug("Saving %s", os_path)
487 487 try:
488 488 if model['type'] == 'notebook':
489 489 self._save_notebook(os_path, model, path)
490 490 elif model['type'] == 'file':
491 491 self._save_file(os_path, model, path)
492 492 elif model['type'] == 'directory':
493 493 self._save_directory(os_path, model, path)
494 494 else:
495 495 raise web.HTTPError(400, "Unhandled contents type: %s" % model['type'])
496 496 except web.HTTPError:
497 497 raise
498 498 except Exception as e:
499 499 self.log.error(u'Error while saving file: %s %s', path, e, exc_info=True)
500 500 raise web.HTTPError(500, u'Unexpected error while saving file: %s %s' % (path, e))
501 501
502 502 validation_message = None
503 503 if model['type'] == 'notebook':
504 504 self.validate_notebook_model(model)
505 505 validation_message = model.get('message', None)
506 506
507 507 model = self.get(path, content=False)
508 508 if validation_message:
509 509 model['message'] = validation_message
510 510
511 511 self.run_post_save_hook(model=model, os_path=os_path)
512 512
513 513 return model
514 514
515 def update(self, model, path):
516 """Update the file's path
517
518 For use in PATCH requests, to enable renaming a file without
519 re-uploading its contents. Only used for renaming at the moment.
520 """
521 path = path.strip('/')
522 new_path = model.get('path', path).strip('/')
523 if path != new_path:
524 self.rename(path, new_path)
525 model = self.get(new_path, content=False)
526 return model
527
528 515 def delete(self, path):
529 516 """Delete file at path."""
530 517 path = path.strip('/')
531 518 os_path = self._get_os_path(path)
532 519 rm = os.unlink
533 520 if os.path.isdir(os_path):
534 521 listing = os.listdir(os_path)
535 522 # don't delete non-empty directories (checkpoints dir doesn't count)
536 523 if listing and listing != [self.checkpoint_dir]:
537 524 raise web.HTTPError(400, u'Directory %s not empty' % os_path)
538 525 elif not os.path.isfile(os_path):
539 526 raise web.HTTPError(404, u'File does not exist: %s' % os_path)
540 527
541 528 # clear checkpoints
542 529 for checkpoint in self.list_checkpoints(path):
543 530 checkpoint_id = checkpoint['id']
544 531 cp_path = self.get_checkpoint_path(checkpoint_id, path)
545 532 if os.path.isfile(cp_path):
546 533 self.log.debug("Unlinking checkpoint %s", cp_path)
547 534 with self.perm_to_403():
548 535 rm(cp_path)
549 536
550 537 if os.path.isdir(os_path):
551 538 self.log.debug("Removing directory %s", os_path)
552 539 with self.perm_to_403():
553 540 shutil.rmtree(os_path)
554 541 else:
555 542 self.log.debug("Unlinking file %s", os_path)
556 543 with self.perm_to_403():
557 544 rm(os_path)
558 545
559 546 def rename(self, old_path, new_path):
560 547 """Rename a file."""
561 548 old_path = old_path.strip('/')
562 549 new_path = new_path.strip('/')
563 550 if new_path == old_path:
564 551 return
565 552
566 553 new_os_path = self._get_os_path(new_path)
567 554 old_os_path = self._get_os_path(old_path)
568 555
569 556 # Should we proceed with the move?
570 557 if os.path.exists(new_os_path):
571 558 raise web.HTTPError(409, u'File already exists: %s' % new_path)
572 559
573 560 # Move the file
574 561 try:
575 562 with self.perm_to_403():
576 563 shutil.move(old_os_path, new_os_path)
577 564 except web.HTTPError:
578 565 raise
579 566 except Exception as e:
580 567 raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_path, e))
581 568
582 569 # Move the checkpoints
583 570 old_checkpoints = self.list_checkpoints(old_path)
584 571 for cp in old_checkpoints:
585 572 checkpoint_id = cp['id']
586 573 old_cp_path = self.get_checkpoint_path(checkpoint_id, old_path)
587 574 new_cp_path = self.get_checkpoint_path(checkpoint_id, new_path)
588 575 if os.path.isfile(old_cp_path):
589 576 self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path)
590 577 with self.perm_to_403():
591 578 shutil.move(old_cp_path, new_cp_path)
592 579
593 580 # Checkpoint-related utilities
594 581
595 582 def get_checkpoint_path(self, checkpoint_id, path):
596 583 """find the path to a checkpoint"""
597 584 path = path.strip('/')
598 585 parent, name = ('/' + path).rsplit('/', 1)
599 586 parent = parent.strip('/')
600 587 basename, ext = os.path.splitext(name)
601 588 filename = u"{name}-{checkpoint_id}{ext}".format(
602 589 name=basename,
603 590 checkpoint_id=checkpoint_id,
604 591 ext=ext,
605 592 )
606 593 os_path = self._get_os_path(path=parent)
607 594 cp_dir = os.path.join(os_path, self.checkpoint_dir)
608 595 with self.perm_to_403():
609 596 ensure_dir_exists(cp_dir)
610 597 cp_path = os.path.join(cp_dir, filename)
611 598 return cp_path
612 599
613 600 def get_checkpoint_model(self, checkpoint_id, path):
614 601 """construct the info dict for a given checkpoint"""
615 602 path = path.strip('/')
616 603 cp_path = self.get_checkpoint_path(checkpoint_id, path)
617 604 stats = os.stat(cp_path)
618 605 last_modified = tz.utcfromtimestamp(stats.st_mtime)
619 606 info = dict(
620 607 id = checkpoint_id,
621 608 last_modified = last_modified,
622 609 )
623 610 return info
624 611
625 612 # public checkpoint API
626 613
627 614 def create_checkpoint(self, path):
628 615 """Create a checkpoint from the current state of a file"""
629 616 path = path.strip('/')
630 617 if not self.file_exists(path):
631 618 raise web.HTTPError(404)
632 619 src_path = self._get_os_path(path)
633 620 # only the one checkpoint ID:
634 621 checkpoint_id = u"checkpoint"
635 622 cp_path = self.get_checkpoint_path(checkpoint_id, path)
636 623 self.log.debug("creating checkpoint for %s", path)
637 624 with self.perm_to_403():
638 625 self._copy(src_path, cp_path)
639 626
640 627 # return the checkpoint info
641 628 return self.get_checkpoint_model(checkpoint_id, path)
642 629
643 630 def list_checkpoints(self, path):
644 631 """list the checkpoints for a given file
645 632
646 633 This contents manager currently only supports one checkpoint per file.
647 634 """
648 635 path = path.strip('/')
649 636 checkpoint_id = "checkpoint"
650 637 os_path = self.get_checkpoint_path(checkpoint_id, path)
651 638 if not os.path.exists(os_path):
652 639 return []
653 640 else:
654 641 return [self.get_checkpoint_model(checkpoint_id, path)]
655 642
656 643
657 644 def restore_checkpoint(self, checkpoint_id, path):
658 645 """restore a file to a checkpointed state"""
659 646 path = path.strip('/')
660 647 self.log.info("restoring %s from checkpoint %s", path, checkpoint_id)
661 648 nb_path = self._get_os_path(path)
662 649 cp_path = self.get_checkpoint_path(checkpoint_id, path)
663 650 if not os.path.isfile(cp_path):
664 651 self.log.debug("checkpoint file does not exist: %s", cp_path)
665 652 raise web.HTTPError(404,
666 653 u'checkpoint does not exist: %s@%s' % (path, checkpoint_id)
667 654 )
668 655 # ensure notebook is readable (never restore from an unreadable notebook)
669 656 if cp_path.endswith('.ipynb'):
670 657 with self.open(cp_path, 'r', encoding='utf-8') as f:
671 658 nbformat.read(f, as_version=4)
672 659 self.log.debug("copying %s -> %s", cp_path, nb_path)
673 660 with self.perm_to_403():
674 661 self._copy(cp_path, nb_path)
675 662
676 663 def delete_checkpoint(self, checkpoint_id, path):
677 664 """delete a file's checkpoint"""
678 665 path = path.strip('/')
679 666 cp_path = self.get_checkpoint_path(checkpoint_id, path)
680 667 if not os.path.isfile(cp_path):
681 668 raise web.HTTPError(404,
682 669 u'Checkpoint does not exist: %s@%s' % (path, checkpoint_id)
683 670 )
684 671 self.log.debug("unlinking %s", cp_path)
685 672 os.unlink(cp_path)
686 673
687 674 def info_string(self):
688 675 return "Serving notebooks from local directory: %s" % self.root_dir
689 676
690 677 def get_kernel_path(self, path, model=None):
691 678 """Return the initial API path of a kernel associated with a given notebook"""
692 679 if '/' in path:
693 680 parent_dir = path.rsplit('/', 1)[0]
694 681 else:
695 682 parent_dir = ''
696 683 return parent_dir
@@ -1,428 +1,433 b''
1 1 """A base class for contents managers."""
2 2
3 3 # Copyright (c) IPython Development Team.
4 4 # Distributed under the terms of the Modified BSD License.
5 5
6 6 from fnmatch import fnmatch
7 7 import itertools
8 8 import json
9 9 import os
10 10 import re
11 11
12 12 from tornado.web import HTTPError
13 13
14 14 from IPython.config.configurable import LoggingConfigurable
15 15 from IPython.nbformat import sign, validate, ValidationError
16 16 from IPython.nbformat.v4 import new_notebook
17 17 from IPython.utils.importstring import import_item
18 18 from IPython.utils.traitlets import Instance, Unicode, List, Any, TraitError
19 19 from IPython.utils.py3compat import string_types
20 20
21 21 copy_pat = re.compile(r'\-Copy\d*\.')
22 22
23 23 class ContentsManager(LoggingConfigurable):
24 24 """Base class for serving files and directories.
25 25
26 26 This serves any text or binary file,
27 27 as well as directories,
28 28 with special handling for JSON notebook documents.
29 29
30 30 Most APIs take a path argument,
31 31 which is always an API-style unicode path,
32 32 and always refers to a directory.
33 33
34 34 - unicode, not url-escaped
35 35 - '/'-separated
36 36 - leading and trailing '/' will be stripped
37 37 - if unspecified, path defaults to '',
38 38 indicating the root path.
39 39
40 40 """
41 41
42 42 notary = Instance(sign.NotebookNotary)
43 43 def _notary_default(self):
44 44 return sign.NotebookNotary(parent=self)
45 45
46 46 hide_globs = List(Unicode, [
47 47 u'__pycache__', '*.pyc', '*.pyo',
48 48 '.DS_Store', '*.so', '*.dylib', '*~',
49 49 ], config=True, help="""
50 50 Glob patterns to hide in file and directory listings.
51 51 """)
52 52
53 53 untitled_notebook = Unicode("Untitled", config=True,
54 54 help="The base name used when creating untitled notebooks."
55 55 )
56 56
57 57 untitled_file = Unicode("untitled", config=True,
58 58 help="The base name used when creating untitled files."
59 59 )
60 60
61 61 untitled_directory = Unicode("Untitled Folder", config=True,
62 62 help="The base name used when creating untitled directories."
63 63 )
64 64
65 65 pre_save_hook = Any(None, config=True,
66 66 help="""Python callable or importstring thereof
67 67
68 68 To be called on a contents model prior to save.
69 69
70 70 This can be used to process the structure,
71 71 such as removing notebook outputs or other side effects that
72 72 should not be saved.
73 73
74 74 It will be called as (all arguments passed by keyword):
75 75
76 76 hook(path=path, model=model, contents_manager=self)
77 77
78 78 model: the model to be saved. Includes file contents.
79 79 modifying this dict will affect the file that is stored.
80 80 path: the API path of the save destination
81 81 contents_manager: this ContentsManager instance
82 82 """
83 83 )
84 84 def _pre_save_hook_changed(self, name, old, new):
85 85 if new and isinstance(new, string_types):
86 86 self.pre_save_hook = import_item(self.pre_save_hook)
87 87 elif new:
88 88 if not callable(new):
89 89 raise TraitError("pre_save_hook must be callable")
90 90
91 91 def run_pre_save_hook(self, model, path, **kwargs):
92 92 """Run the pre-save hook if defined, and log errors"""
93 93 if self.pre_save_hook:
94 94 try:
95 95 self.log.debug("Running pre-save hook on %s", path)
96 96 self.pre_save_hook(model=model, path=path, contents_manager=self, **kwargs)
97 97 except Exception:
98 98 self.log.error("Pre-save hook failed on %s", path, exc_info=True)
99 99
100 100 # ContentsManager API part 1: methods that must be
101 101 # implemented in subclasses.
102 102
103 103 def dir_exists(self, path):
104 104 """Does the API-style path (directory) actually exist?
105 105
106 106 Like os.path.isdir
107 107
108 108 Override this method in subclasses.
109 109
110 110 Parameters
111 111 ----------
112 112 path : string
113 113 The path to check
114 114
115 115 Returns
116 116 -------
117 117 exists : bool
118 118 Whether the path does indeed exist.
119 119 """
120 120 raise NotImplementedError
121 121
122 122 def is_hidden(self, path):
123 123 """Does the API style path correspond to a hidden directory or file?
124 124
125 125 Parameters
126 126 ----------
127 127 path : string
128 128 The path to check. This is an API path (`/` separated,
129 129 relative to root dir).
130 130
131 131 Returns
132 132 -------
133 133 hidden : bool
134 134 Whether the path is hidden.
135 135
136 136 """
137 137 raise NotImplementedError
138 138
139 139 def file_exists(self, path=''):
140 140 """Does a file exist at the given path?
141 141
142 142 Like os.path.isfile
143 143
144 144 Override this method in subclasses.
145 145
146 146 Parameters
147 147 ----------
148 148 name : string
149 149 The name of the file you are checking.
150 150 path : string
151 151 The relative path to the file's directory (with '/' as separator)
152 152
153 153 Returns
154 154 -------
155 155 exists : bool
156 156 Whether the file exists.
157 157 """
158 158 raise NotImplementedError('must be implemented in a subclass')
159 159
160 160 def exists(self, path):
161 161 """Does a file or directory exist at the given path?
162 162
163 163 Like os.path.exists
164 164
165 165 Parameters
166 166 ----------
167 167 path : string
168 168 The relative path to the file's directory (with '/' as separator)
169 169
170 170 Returns
171 171 -------
172 172 exists : bool
173 173 Whether the target exists.
174 174 """
175 175 return self.file_exists(path) or self.dir_exists(path)
176 176
177 177 def get(self, path, content=True, type=None, format=None):
178 178 """Get the model of a file or directory with or without content."""
179 179 raise NotImplementedError('must be implemented in a subclass')
180 180
181 181 def save(self, model, path):
182 182 """Save the file or directory and return the model with no content.
183 183
184 184 Save implementations should call self.run_pre_save_hook(model=model, path=path)
185 185 prior to writing any data.
186 186 """
187 187 raise NotImplementedError('must be implemented in a subclass')
188 188
189 def update(self, model, path):
190 """Update the file or directory and return the model with no content.
191
192 For use in PATCH requests, to enable renaming a file without
193 re-uploading its contents. Only used for renaming at the moment.
194 """
195 raise NotImplementedError('must be implemented in a subclass')
196
197 189 def delete(self, path):
198 190 """Delete file or directory by path."""
199 191 raise NotImplementedError('must be implemented in a subclass')
200 192
201 193 def create_checkpoint(self, path):
202 194 """Create a checkpoint of the current state of a file
203 195
204 196 Returns a checkpoint_id for the new checkpoint.
205 197 """
206 198 raise NotImplementedError("must be implemented in a subclass")
207 199
208 200 def list_checkpoints(self, path):
209 201 """Return a list of checkpoints for a given file"""
210 202 return []
211 203
212 204 def restore_checkpoint(self, checkpoint_id, path):
213 205 """Restore a file from one of its checkpoints"""
214 206 raise NotImplementedError("must be implemented in a subclass")
215 207
216 208 def delete_checkpoint(self, checkpoint_id, path):
217 209 """delete a checkpoint for a file"""
218 210 raise NotImplementedError("must be implemented in a subclass")
219 211
220 212 # ContentsManager API part 2: methods that have useable default
221 213 # implementations, but can be overridden in subclasses.
222 214
215 def update(self, model, path):
216 """Update the file's path
217
218 For use in PATCH requests, to enable renaming a file without
219 re-uploading its contents. Only used for renaming at the moment.
220 """
221 path = path.strip('/')
222 new_path = model.get('path', path).strip('/')
223 if path != new_path:
224 self.rename(path, new_path)
225 model = self.get(new_path, content=False)
226 return model
227
223 228 def info_string(self):
224 229 return "Serving contents"
225 230
226 231 def get_kernel_path(self, path, model=None):
227 232 """Return the API path for the kernel
228 233
229 234 KernelManagers can turn this value into a filesystem path,
230 235 or ignore it altogether.
231 236
232 237 The default value here will start kernels in the directory of the
233 238 notebook server. FileContentsManager overrides this to use the
234 239 directory containing the notebook.
235 240 """
236 241 return ''
237 242
238 243 def increment_filename(self, filename, path='', insert=''):
239 244 """Increment a filename until it is unique.
240 245
241 246 Parameters
242 247 ----------
243 248 filename : unicode
244 249 The name of a file, including extension
245 250 path : unicode
246 251 The API path of the target's directory
247 252
248 253 Returns
249 254 -------
250 255 name : unicode
251 256 A filename that is unique, based on the input filename.
252 257 """
253 258 path = path.strip('/')
254 259 basename, ext = os.path.splitext(filename)
255 260 for i in itertools.count():
256 261 if i:
257 262 insert_i = '{}{}'.format(insert, i)
258 263 else:
259 264 insert_i = ''
260 265 name = u'{basename}{insert}{ext}'.format(basename=basename,
261 266 insert=insert_i, ext=ext)
262 267 if not self.exists(u'{}/{}'.format(path, name)):
263 268 break
264 269 return name
265 270
266 271 def validate_notebook_model(self, model):
267 272 """Add failed-validation message to model"""
268 273 try:
269 274 validate(model['content'])
270 275 except ValidationError as e:
271 276 model['message'] = u'Notebook Validation failed: {}:\n{}'.format(
272 277 e.message, json.dumps(e.instance, indent=1, default=lambda obj: '<UNKNOWN>'),
273 278 )
274 279 return model
275 280
276 281 def new_untitled(self, path='', type='', ext=''):
277 282 """Create a new untitled file or directory in path
278 283
279 284 path must be a directory
280 285
281 286 File extension can be specified.
282 287
283 288 Use `new` to create files with a fully specified path (including filename).
284 289 """
285 290 path = path.strip('/')
286 291 if not self.dir_exists(path):
287 292 raise HTTPError(404, 'No such directory: %s' % path)
288 293
289 294 model = {}
290 295 if type:
291 296 model['type'] = type
292 297
293 298 if ext == '.ipynb':
294 299 model.setdefault('type', 'notebook')
295 300 else:
296 301 model.setdefault('type', 'file')
297 302
298 303 insert = ''
299 304 if model['type'] == 'directory':
300 305 untitled = self.untitled_directory
301 306 insert = ' '
302 307 elif model['type'] == 'notebook':
303 308 untitled = self.untitled_notebook
304 309 ext = '.ipynb'
305 310 elif model['type'] == 'file':
306 311 untitled = self.untitled_file
307 312 else:
308 313 raise HTTPError(400, "Unexpected model type: %r" % model['type'])
309 314
310 315 name = self.increment_filename(untitled + ext, path, insert=insert)
311 316 path = u'{0}/{1}'.format(path, name)
312 317 return self.new(model, path)
313 318
314 319 def new(self, model=None, path=''):
315 320 """Create a new file or directory and return its model with no content.
316 321
317 322 To create a new untitled entity in a directory, use `new_untitled`.
318 323 """
319 324 path = path.strip('/')
320 325 if model is None:
321 326 model = {}
322 327
323 328 if path.endswith('.ipynb'):
324 329 model.setdefault('type', 'notebook')
325 330 else:
326 331 model.setdefault('type', 'file')
327 332
328 333 # no content, not a directory, so fill out new-file model
329 334 if 'content' not in model and model['type'] != 'directory':
330 335 if model['type'] == 'notebook':
331 336 model['content'] = new_notebook()
332 337 model['format'] = 'json'
333 338 else:
334 339 model['content'] = ''
335 340 model['type'] = 'file'
336 341 model['format'] = 'text'
337 342
338 343 model = self.save(model, path)
339 344 return model
340 345
341 346 def copy(self, from_path, to_path=None):
342 347 """Copy an existing file and return its new model.
343 348
344 349 If to_path not specified, it will be the parent directory of from_path.
345 350 If to_path is a directory, filename will increment `from_path-Copy#.ext`.
346 351
347 352 from_path must be a full path to a file.
348 353 """
349 354 path = from_path.strip('/')
350 355 if to_path is not None:
351 356 to_path = to_path.strip('/')
352 357
353 358 if '/' in path:
354 359 from_dir, from_name = path.rsplit('/', 1)
355 360 else:
356 361 from_dir = ''
357 362 from_name = path
358 363
359 364 model = self.get(path)
360 365 model.pop('path', None)
361 366 model.pop('name', None)
362 367 if model['type'] == 'directory':
363 368 raise HTTPError(400, "Can't copy directories")
364 369
365 370 if to_path is None:
366 371 to_path = from_dir
367 372 if self.dir_exists(to_path):
368 373 name = copy_pat.sub(u'.', from_name)
369 374 to_name = self.increment_filename(name, to_path, insert='-Copy')
370 375 to_path = u'{0}/{1}'.format(to_path, to_name)
371 376
372 377 model = self.save(model, to_path)
373 378 return model
374 379
375 380 def log_info(self):
376 381 self.log.info(self.info_string())
377 382
378 383 def trust_notebook(self, path):
379 384 """Explicitly trust a notebook
380 385
381 386 Parameters
382 387 ----------
383 388 path : string
384 389 The path of a notebook
385 390 """
386 391 model = self.get(path)
387 392 nb = model['content']
388 393 self.log.warn("Trusting notebook %s", path)
389 394 self.notary.mark_cells(nb, True)
390 395 self.save(model, path)
391 396
392 397 def check_and_sign(self, nb, path=''):
393 398 """Check for trusted cells, and sign the notebook.
394 399
395 400 Called as a part of saving notebooks.
396 401
397 402 Parameters
398 403 ----------
399 404 nb : dict
400 405 The notebook dict
401 406 path : string
402 407 The notebook's path (for logging)
403 408 """
404 409 if self.notary.check_cells(nb):
405 410 self.notary.sign(nb)
406 411 else:
407 412 self.log.warn("Saving untrusted notebook %s", path)
408 413
409 414 def mark_trusted_cells(self, nb, path=''):
410 415 """Mark cells as trusted if the notebook signature matches.
411 416
412 417 Called as a part of loading notebooks.
413 418
414 419 Parameters
415 420 ----------
416 421 nb : dict
417 422 The notebook object (in current nbformat)
418 423 path : string
419 424 The notebook's path (for logging)
420 425 """
421 426 trusted = self.notary.check_signature(nb)
422 427 if not trusted:
423 428 self.log.warn("Notebook %s is not trusted", path)
424 429 self.notary.mark_cells(nb, trusted)
425 430
426 431 def should_list(self, name):
427 432 """Should this file/directory name be displayed in a listing?"""
428 433 return not any(fnmatch(name, glob) for glob in self.hide_globs)
General Comments 0
You need to be logged in to leave comments. Login now