diff --git a/IPython/html/base/handlers.py b/IPython/html/base/handlers.py index 8796e55..76f9164 100644 --- a/IPython/html/base/handlers.py +++ b/IPython/html/base/handlers.py @@ -416,6 +416,8 @@ class TrailingSlashHandler(web.RequestHandler): path_regex = r"(?P(?:/.*)*)" notebook_name_regex = r"(?P[^/]+\.ipynb)" notebook_path_regex = "%s/%s" % (path_regex, notebook_name_regex) +file_name_regex = r"(?P[^/]+)" +file_path_regex = "%s/%s" % (path_regex, file_name_regex) #----------------------------------------------------------------------------- # URL to handler mappings diff --git a/IPython/html/nbconvert/handlers.py b/IPython/html/nbconvert/handlers.py index 180e6c6..93e6bf1 100644 --- a/IPython/html/nbconvert/handlers.py +++ b/IPython/html/nbconvert/handlers.py @@ -1,3 +1,8 @@ +"""Tornado handlers for nbconvert.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + import io import os import zipfile @@ -73,7 +78,7 @@ class NbconvertFileHandler(IPythonHandler): exporter = get_exporter(format, config=self.config, log=self.log) path = path.strip('/') - model = self.contents_manager.get(name=name, path=path) + model = self.contents_manager.get_model(name=name, path=path) self.set_header('Last-Modified', model['last_modified']) diff --git a/IPython/html/services/contents/filemanager.py b/IPython/html/services/contents/filemanager.py index 2ddca80..e4a0b59 100644 --- a/IPython/html/services/contents/filemanager.py +++ b/IPython/html/services/contents/filemanager.py @@ -3,6 +3,7 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import base64 import io import os import glob @@ -56,17 +57,29 @@ class FileContentsManager(ContentsManager): except OSError as e: self.log.debug("copystat on %s failed", dest, exc_info=True) - def get_names(self, path=''): - """List all filenames in the path (relative to root_dir).""" - path = path.strip('/') - if not os.path.isdir(self._get_os_path(path=path)): - raise web.HTTPError(404, 'Directory not found: ' + path) - names = glob.glob(self._get_os_path('*', path)) - names = [ os.path.basename(name) for name in names if os.path.isfile(name)] - return names + def _get_os_path(self, name=None, path=''): + """Given a filename and a URL path, return its file system + path. + + Parameters + ---------- + name : string + A filename + path : string + The relative URL path (with '/' as separator) to the named + file. + + Returns + ------- + path : string + API path to be evaluated relative to root_dir. + """ + if name is not None: + path = path + '/' + name + return to_os_path(path, self.root_dir) def path_exists(self, path): - """Does the API-style path (directory) actually exist? + """Does the API-style path refer to an extant directory? Parameters ---------- @@ -102,29 +115,26 @@ class FileContentsManager(ContentsManager): os_path = self._get_os_path(path=path) return is_hidden(os_path, self.root_dir) - def _get_os_path(self, name=None, path=''): - """Given a filename and a URL path, return its file system - path. + def file_exists(self, name, path=''): + """Returns True if the file exists, else returns False. Parameters ---------- name : string - A filename + The name of the file you are checking. path : string - The relative URL path (with '/' as separator) to the named - file. + The relative path to the file's directory (with '/' as separator) Returns ------- - path : string - API path to be evaluated relative to root_dir. + bool """ - if name is not None: - path = path + '/' + name - return to_os_path(path, self.root_dir) + path = path.strip('/') + nbpath = self._get_os_path(name, path=path) + return os.path.isfile(nbpath) - def file_exists(self, name, path=''): - """Returns a True if the file exists, else returns False. + def exists(self, name=None, path=''): + """Returns True if the path [and name] exists, else returns False. Parameters ---------- @@ -138,83 +148,107 @@ class FileContentsManager(ContentsManager): bool """ path = path.strip('/') - nbpath = self._get_os_path(name, path=path) - return os.path.isfile(nbpath) + os_path = self._get_os_path(name, path=path) + return os.path.exists(os_path) - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def list_dirs(self, path): - """List the directories for a given API style path.""" - path = path.strip('/') - os_path = self._get_os_path('', path) - if not os.path.isdir(os_path): - raise web.HTTPError(404, u'directory does not exist: %r' % os_path) - elif is_hidden(os_path, self.root_dir): - self.log.info("Refusing to serve hidden directory, via 404 Error") - raise web.HTTPError(404, u'directory does not exist: %r' % os_path) - dir_names = os.listdir(os_path) - dirs = [] - for name in dir_names: - os_path = self._get_os_path(name, path) - if os.path.isdir(os_path) and not is_hidden(os_path, self.root_dir)\ - and self.should_list(name): - try: - model = self.get_dir_model(name, path) - except IOError: - pass - dirs.append(model) - dirs = sorted(dirs, key=sort_key) - return dirs - - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def get_dir_model(self, name, path=''): - """Get the directory model given a directory name and its API style path""" - path = path.strip('/') + def _base_model(self, name, path=''): + """Build the common base of a contents model""" os_path = self._get_os_path(name, path) - if not os.path.isdir(os_path): - raise IOError('directory does not exist: %r' % os_path) info = os.stat(os_path) last_modified = tz.utcfromtimestamp(info.st_mtime) created = tz.utcfromtimestamp(info.st_ctime) # Create the notebook model. - model ={} + model = {} model['name'] = name model['path'] = path model['last_modified'] = last_modified model['created'] = created + model['content'] = None + model['format'] = None + return model + + def _dir_model(self, name, path='', content=True): + """Build a model for a directory + + if content is requested, will include a listing of the directory + """ + os_path = self._get_os_path(name, path) + + if not os.path.isdir(os_path): + raise web.HTTPError(404, u'directory does not exist: %r' % os_path) + elif is_hidden(os_path, self.root_dir): + self.log.info("Refusing to serve hidden directory, via 404 Error") + raise web.HTTPError(404, u'directory does not exist: %r' % os_path) + + if name is None: + if '/' in path: + path, name = path.rsplit('/', 1) + else: + name = '' + model = self._base_model(name, path) model['type'] = 'directory' + dir_path = u'{}/{}'.format(path, name) + if content: + contents = [] + for os_path in glob.glob(self._get_os_path('*', dir_path)): + name = os.path.basename(os_path) + if self.should_list(name) and not is_hidden(os_path, self.root_dir): + contents.append(self.get_model(name=name, path=dir_path, content=False)) + + model['content'] = sorted(contents, key=sort_key) + return model - def list_files(self, path): - """Returns a list of dictionaries that are the standard model - for all notebooks in the relative 'path'. + def _file_model(self, name, path='', content=True): + """Build a model for a file - Parameters - ---------- - path : str - the URL path that describes the relative path for the - listed notebooks + if content is requested, include the file contents. + Text files will be unicode, binary files will be base64-encoded. + """ + model = self._base_model(name, path) + model['type'] = 'file' + if content: + os_path = self._get_os_path(name, path) + try: + with io.open(os_path, 'r', encoding='utf-8') as f: + model['content'] = f.read() + except UnicodeError as e: + with io.open(os_path, 'rb') as f: + bcontent = f.read() + model['content'] = base64.encodestring(bcontent).decode('ascii') + model['format'] = 'base64' + else: + model['format'] = 'text' + return model - Returns - ------- - notebooks : list of dicts - a list of the notebook models without 'content' + + def _notebook_model(self, name, path='', content=True): + """Build a notebook model + + if content is requested, the notebook content will be populated + as a JSON structure (not double-serialized) """ - path = path.strip('/') - names = self.get_names(path) - notebooks = [self.get(name, path, content=False) - for name in names if self.should_list(name)] - notebooks = sorted(notebooks, key=sort_key) - return notebooks + model = self._base_model(name, path) + model['type'] = 'notebook' + if content: + os_path = self._get_os_path(name, path) + with io.open(os_path, 'r', encoding='utf-8') as f: + try: + nb = current.read(f, u'json') + except Exception as e: + raise web.HTTPError(400, u"Unreadable Notebook: %s %s" % (os_path, e)) + self.mark_trusted_cells(nb, name, path) + model['content'] = nb + model['format'] = 'json' + return model - def get(self, name, path='', content=True): - """ Takes a path and name for a notebook and returns its model + def get_model(self, name, path='', content=True): + """ Takes a path and name for an entity and returns its model Parameters ---------- name : str - the name of the notebook + the name of the target path : str the URL path that describes the relative path for the notebook @@ -222,31 +256,21 @@ class FileContentsManager(ContentsManager): Returns ------- model : dict - the notebook model. If contents=True, returns the 'contents' - dict in the model as well. + the contents model. If content=True, returns the contents + of the file or directory as well. """ path = path.strip('/') - if not self.file_exists(name=name, path=path): - raise web.HTTPError(404, u'Notebook does not exist: %s' % name) + + if not self.exists(name=name, path=path): + raise web.HTTPError(404, u'No such file or directory: %s/%s' % (path, name)) + os_path = self._get_os_path(name, path) - info = os.stat(os_path) - last_modified = tz.utcfromtimestamp(info.st_mtime) - created = tz.utcfromtimestamp(info.st_ctime) - # Create the notebook model. - model ={} - model['name'] = name - model['path'] = path - model['last_modified'] = last_modified - model['created'] = created - model['type'] = 'notebook' - if content: - with io.open(os_path, 'r', encoding='utf-8') as f: - try: - nb = current.read(f, u'json') - except Exception as e: - raise web.HTTPError(400, u"Unreadable Notebook: %s %s" % (os_path, e)) - self.mark_trusted_cells(nb, name, path) - model['content'] = nb + if os.path.isdir(os_path): + model = self._dir_model(name, path, content) + elif name.endswith('.ipynb'): + model = self._notebook_model(name, path, content) + else: + model = self._file_model(name, path, content) return model def save(self, model, name='', path=''): @@ -281,7 +305,7 @@ class FileContentsManager(ContentsManager): except Exception as e: raise web.HTTPError(400, u'Unexpected error while autosaving notebook: %s %s' % (os_path, e)) - model = self.get(new_name, new_path, content=False) + model = self.get_model(new_name, new_path, content=False) return model def update(self, model, name, path=''): @@ -291,7 +315,7 @@ class FileContentsManager(ContentsManager): new_path = model.get('path', path).strip('/') if path != new_path or name != new_name: self.rename(name, path, new_name, new_path) - model = self.get(new_name, new_path, content=False) + model = self.get_model(new_name, new_path, content=False) return model def delete(self, name, path=''): diff --git a/IPython/html/services/contents/handlers.py b/IPython/html/services/contents/handlers.py index 878b8e7..e649525 100644 --- a/IPython/html/services/contents/handlers.py +++ b/IPython/html/services/contents/handlers.py @@ -11,15 +11,15 @@ from IPython.html.utils import url_path_join, url_escape from IPython.utils.jsonutil import date_default from IPython.html.base.handlers import (IPythonHandler, json_errors, - notebook_path_regex, path_regex, - notebook_name_regex) + file_path_regex, path_regex, + file_name_regex) class ContentsHandler(IPythonHandler): SUPPORTED_METHODS = (u'GET', u'PUT', u'PATCH', u'POST', u'DELETE') - def location_url(self, name, path=''): + def location_url(self, name, path): """Return the full URL location of a file. Parameters @@ -49,25 +49,19 @@ class ContentsHandler(IPythonHandler): * GET with path and no filename lists files in a directory * GET with path and filename returns file contents model """ - cm = self.contents_manager - # Check to see if a filename was given - if name is None: - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. This should only handle notebooks - # and not directories. - dirs = cm.list_dirs(path) + path = path or '' + model = self.contents_manager.get_model(name=name, path=path) + if model['type'] == 'directory': + # resort listing to group directories at the top + dirs = [] files = [] - index = [] - for nb in cm.list_files(path): - if nb['name'].lower() == 'index.ipynb': - index.append(nb) + for entry in model['content']: + if entry['type'] == 'directory': + dirs.append(entry) else: - files.append(nb) - files = index + dirs + files - self.finish(json.dumps(files, default=date_default)) - return - # get and return notebook representation - model = cm.get(name, path) + # do we also want to group notebooks separate from files? + files.append(entry) + model['content'] = dirs + files self._finish_model(model, location=False) @web.authenticated @@ -148,8 +142,16 @@ class ContentsHandler(IPythonHandler): """ if name is not None: + path = u'{}/{}'.format(path, name) + + cm = self.contents_manager + + if cm.file_exists(path): raise web.HTTPError(400, "Only POST to directories. Use PUT for full names.") + if not cm.path_exists(path): + raise web.HTTPError(404, "No such directory: %s" % path) + model = self.get_json_body() if model is not None: @@ -200,6 +202,7 @@ class ContentsHandler(IPythonHandler): def delete(self, path='', name=None): """delete a file in the given path""" cm = self.contents_manager + self.log.warn('delete %s:%s', path, name) cm.delete(name, path) self.set_status(204) self.finish() @@ -262,9 +265,9 @@ class ModifyCheckpointsHandler(IPythonHandler): _checkpoint_id_regex = r"(?P[\w-]+)" default_handlers = [ - (r"/api/contents%s/checkpoints" % notebook_path_regex, CheckpointsHandler), - (r"/api/contents%s/checkpoints/%s" % (notebook_path_regex, _checkpoint_id_regex), + (r"/api/contents%s/checkpoints" % file_path_regex, CheckpointsHandler), + (r"/api/contents%s/checkpoints/%s" % (file_path_regex, _checkpoint_id_regex), ModifyCheckpointsHandler), - (r"/api/contents%s" % notebook_path_regex, ContentsHandler), + (r"/api/contents%s" % file_path_regex, ContentsHandler), (r"/api/contents%s" % path_regex, ContentsHandler), ] diff --git a/IPython/html/services/contents/manager.py b/IPython/html/services/contents/manager.py index 32f3677..dff77b5 100644 --- a/IPython/html/services/contents/manager.py +++ b/IPython/html/services/contents/manager.py @@ -75,27 +75,7 @@ class ContentsManager(LoggingConfigurable): """ raise NotImplementedError('must be implemented in a subclass') - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def list_dirs(self, path): - """List the directory models for a given API style path.""" - raise NotImplementedError('must be implemented in a subclass') - - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def get_dir_model(self, name, path=''): - """Get the directory model given a directory name and its API style path. - - The keys in the model should be: - * name - * path - * last_modified - * created - * type='directory' - """ - raise NotImplementedError('must be implemented in a subclass') - - def list_files(self, path=''): + def list(self, path=''): """Return a list of contents dicts without content. This returns a list of dicts @@ -196,7 +176,7 @@ class ContentsManager(LoggingConfigurable): If to_name not specified, increment `from_name-Copy#.ipynb`. """ path = path.strip('/') - model = self.get(from_name, path) + model = self.get_model(from_name, path) if not to_name: base, ext = os.path.splitext(from_name) copy_name = u'{0}-Copy{1}'.format(base, ext) @@ -218,7 +198,7 @@ class ContentsManager(LoggingConfigurable): path : string The notebook's directory """ - model = self.get(name, path) + model = self.get_model(name, path) nb = model['content'] self.log.warn("Trusting notebook %s/%s", path, name) self.notary.mark_cells(nb, True) diff --git a/IPython/html/services/contents/tests/test_contents_api.py b/IPython/html/services/contents/tests/test_contents_api.py index 256b234..5381b82 100644 --- a/IPython/html/services/contents/tests/test_contents_api.py +++ b/IPython/html/services/contents/tests/test_contents_api.py @@ -1,6 +1,7 @@ # coding: utf-8 """Test the contents webservice API.""" +import base64 import io import json import os @@ -23,11 +24,11 @@ from IPython.utils.data import uniq_stable # TODO: Remove this after we create the contents web service and directories are # no longer listed by the notebook web service. -def notebooks_only(nb_list): - return [nb for nb in nb_list if nb['type']=='notebook'] +def notebooks_only(dir_model): + return [nb for nb in dir_model['content'] if nb['type']=='notebook'] -def dirs_only(nb_list): - return [x for x in nb_list if x['type']=='directory'] +def dirs_only(dir_model): + return [x for x in dir_model['content'] if x['type']=='directory'] class API(object): @@ -112,8 +113,20 @@ class APITest(NotebookTestBase): del dirs[0] # remove '' top_level_dirs = {normalize('NFC', d.split('/')[0]) for d in dirs} + @staticmethod + def _blob_for_name(name): + return name.encode('utf-8') + b'\xFF' + + @staticmethod + def _txt_for_name(name): + return u'%s text file' % name + def setUp(self): nbdir = self.notebook_dir.name + self.blob = os.urandom(100) + self.b64_blob = base64.encodestring(self.blob).decode('ascii') + + for d in (self.dirs + self.hidden_dirs): d.replace('/', os.sep) @@ -122,11 +135,21 @@ class APITest(NotebookTestBase): for d, name in self.dirs_nbs: d = d.replace('/', os.sep) + # create a notebook with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w', encoding='utf-8') as f: nb = new_notebook(name=name) write(nb, f, format='ipynb') + # create a text file + with io.open(pjoin(nbdir, d, '%s.txt' % name), 'w', + encoding='utf-8') as f: + f.write(self._txt_for_name(name)) + + # create a binary file + with io.open(pjoin(nbdir, d, '%s.blob' % name), 'wb') as f: + f.write(self._blob_for_name(name)) + self.api = API(self.base_url()) def tearDown(self): @@ -178,18 +201,49 @@ class APITest(NotebookTestBase): with assert_http_error(404): self.api.list('nonexistant') - def test_get_contents(self): + def test_get_nb_contents(self): for d, name in self.dirs_nbs: nb = self.api.read('%s.ipynb' % name, d+'/').json() self.assertEqual(nb['name'], u'%s.ipynb' % name) + self.assertEqual(nb['type'], 'notebook') + self.assertIn('content', nb) + self.assertEqual(nb['format'], 'json') self.assertIn('content', nb) self.assertIn('metadata', nb['content']) self.assertIsInstance(nb['content']['metadata'], dict) + def test_get_contents_no_such_file(self): # Name that doesn't exist - should be a 404 with assert_http_error(404): self.api.read('q.ipynb', 'foo') + def test_get_text_file_contents(self): + for d, name in self.dirs_nbs: + model = self.api.read(u'%s.txt' % name, d+'/').json() + self.assertEqual(model['name'], u'%s.txt' % name) + self.assertIn('content', model) + self.assertEqual(model['format'], 'text') + self.assertEqual(model['type'], 'file') + self.assertEqual(model['content'], self._txt_for_name(name)) + + # Name that doesn't exist - should be a 404 + with assert_http_error(404): + self.api.read('q.txt', 'foo') + + def test_get_binary_file_contents(self): + for d, name in self.dirs_nbs: + model = self.api.read(u'%s.blob' % name, d+'/').json() + self.assertEqual(model['name'], u'%s.blob' % name) + self.assertIn('content', model) + self.assertEqual(model['format'], 'base64') + self.assertEqual(model['type'], 'file') + b64_data = base64.encodestring(self._blob_for_name(name)).decode('ascii') + self.assertEqual(model['content'], b64_data) + + # Name that doesn't exist - should be a 404 + with assert_http_error(404): + self.api.read('q.txt', 'foo') + def _check_nb_created(self, resp, name, path): self.assertEqual(resp.status_code, 201) location_header = py3compat.str_to_unicode(resp.headers['Location']) diff --git a/IPython/html/services/contents/tests/test_manager.py b/IPython/html/services/contents/tests/test_manager.py index 8ad9efa..44f7e33 100644 --- a/IPython/html/services/contents/tests/test_manager.py +++ b/IPython/html/services/contents/tests/test_manager.py @@ -70,7 +70,7 @@ class TestFileContentsManager(TestCase): self.assertEqual(cp_subdir, os.path.join(root, subd, fm.checkpoint_dir, cp_name)) -class TestNotebookManager(TestCase): +class TestContentsManager(TestCase): def setUp(self): self._temp_dir = TemporaryDirectory() @@ -105,7 +105,7 @@ class TestNotebookManager(TestCase): name = model['name'] path = model['path'] - full_model = cm.get(name, path) + full_model = cm.get_model(name, path) nb = full_model['content'] self.add_code_cell(nb) @@ -140,7 +140,7 @@ class TestNotebookManager(TestCase): path = model['path'] # Check that we 'get' on the notebook we just created - model2 = cm.get(name, path) + model2 = cm.get_model(name, path) assert isinstance(model2, dict) self.assertIn('name', model2) self.assertIn('path', model2) @@ -151,7 +151,7 @@ class TestNotebookManager(TestCase): sub_dir = '/foo/' self.make_dir(cm.root_dir, 'foo') model = cm.create_notebook(None, sub_dir) - model2 = cm.get(name, sub_dir) + model2 = cm.get_model(name, sub_dir) assert isinstance(model2, dict) self.assertIn('name', model2) self.assertIn('path', model2) @@ -175,7 +175,7 @@ class TestNotebookManager(TestCase): self.assertEqual(model['name'], 'test.ipynb') # Make sure the old name is gone - self.assertRaises(HTTPError, cm.get, name, path) + self.assertRaises(HTTPError, cm.get_model, name, path) # Test in sub-directory # Create a directory and notebook in that directory @@ -195,7 +195,7 @@ class TestNotebookManager(TestCase): self.assertEqual(model['path'], sub_dir.strip('/')) # Make sure the old name is gone - self.assertRaises(HTTPError, cm.get, name, path) + self.assertRaises(HTTPError, cm.get_model, name, path) def test_save(self): cm = self.contents_manager @@ -205,7 +205,7 @@ class TestNotebookManager(TestCase): path = model['path'] # Get the model with 'content' - full_model = cm.get(name, path) + full_model = cm.get_model(name, path) # Save the notebook model = cm.save(full_model, name, path) @@ -222,7 +222,7 @@ class TestNotebookManager(TestCase): model = cm.create_notebook(None, sub_dir) name = model['name'] path = model['path'] - model = cm.get(name, path) + model = cm.get_model(name, path) # Change the name in the model for rename model = cm.save(model, name, path) @@ -241,7 +241,7 @@ class TestNotebookManager(TestCase): cm.delete(name, path) # Check that a 'get' on the deleted notebook raises and error - self.assertRaises(HTTPError, cm.get, name, path) + self.assertRaises(HTTPError, cm.get_model, name, path) def test_copy(self): cm = self.contents_manager @@ -262,12 +262,12 @@ class TestNotebookManager(TestCase): cm = self.contents_manager nb, name, path = self.new_notebook() - untrusted = cm.get(name, path)['content'] + untrusted = cm.get_model(name, path)['content'] assert not cm.notary.check_cells(untrusted) # print(untrusted) cm.trust_notebook(name, path) - trusted = cm.get(name, path)['content'] + trusted = cm.get_model(name, path)['content'] # print(trusted) assert cm.notary.check_cells(trusted) @@ -281,7 +281,7 @@ class TestNotebookManager(TestCase): assert not cell.trusted cm.trust_notebook(name, path) - nb = cm.get(name, path)['content'] + nb = cm.get_model(name, path)['content'] for cell in nb.worksheets[0].cells: if cell.cell_type == 'code': assert cell.trusted @@ -295,7 +295,7 @@ class TestNotebookManager(TestCase): assert not cm.notary.check_signature(nb) cm.trust_notebook(name, path) - nb = cm.get(name, path)['content'] + nb = cm.get_model(name, path)['content'] cm.mark_trusted_cells(nb, name, path) cm.check_and_sign(nb, name, path) assert cm.notary.check_signature(nb) diff --git a/IPython/html/static/tree/js/notebooklist.js b/IPython/html/static/tree/js/notebooklist.js index 7827241..0be114a 100644 --- a/IPython/html/static/tree/js/notebooklist.js +++ b/IPython/html/static/tree/js/notebooklist.js @@ -161,7 +161,8 @@ define([ message = param.msg; } var item = null; - var len = data.length; + var content = data.content; + var len = content.length; this.clear_list(); if (len === 0) { item = this.new_notebook_item(0); @@ -177,12 +178,12 @@ define([ offset = 1; } for (var i=0; i