From 905e751b31fda588b669c1a6ae8940c0fd8677db 2014-08-26 08:25:38 From: Gordon Ball Date: 2014-08-26 08:25:38 Subject: [PATCH] Merge master --- diff --git a/.gitignore b/.gitignore index 37530a6..e227fac 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ docs/source/api/generated docs/source/config/options docs/gh-pages IPython/html/notebook/static/mathjax +IPython/html/static/style/*.map *.py[co] __pycache__ *.egg-info diff --git a/.travis.yml b/.travis.yml index 368445f..7648837 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,8 +13,10 @@ before_install: # Pierre Carrier's PPA for PhantomJS and CasperJS - time sudo add-apt-repository -y ppa:pcarrier/ppa - time sudo apt-get update - - time sudo apt-get install pandoc casperjs nodejs libzmq3-dev - - time pip install -f https://nipy.bic.berkeley.edu/wheelhouse/travis jinja2 sphinx pygments tornado requests mock pyzmq jsonschema jsonpointer + - time sudo apt-get install pandoc casperjs libzmq3-dev + # pin tornado < 4 for js tests while phantom is on super old webkit + - if [[ $GROUP == 'js' ]]; then pip install 'tornado<4'; fi + - time pip install -f https://nipy.bic.berkeley.edu/wheelhouse/travis jinja2 sphinx pygments tornado requests mock pyzmq jsonschema jsonpointer mistune install: - time python setup.py install -q script: diff --git a/IPython/config/configurable.py b/IPython/config/configurable.py index cd661d7..24970cc 100644 --- a/IPython/config/configurable.py +++ b/IPython/config/configurable.py @@ -1,31 +1,11 @@ # encoding: utf-8 -""" -A base class for objects that are configurable. +"""A base class for objects that are configurable.""" -Inheritance diagram: +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. -.. inheritance-diagram:: IPython.config.configurable - :parts: 3 - -Authors: - -* Brian Granger -* Fernando Perez -* Min RK -""" from __future__ import print_function -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - import logging from copy import deepcopy @@ -375,16 +355,12 @@ class LoggingConfigurable(Configurable): """A parent class for Configurables that log. Subclasses have a log trait, and the default behavior - is to get the logger from the currently running Application - via Application.instance().log. + is to get the logger from the currently running Application. """ log = Instance('logging.Logger') def _log_default(self): - from IPython.config.application import Application - if Application.initialized(): - return Application.instance().log - else: - return logging.getLogger() + from IPython.utils import log + return log.get_logger() diff --git a/IPython/config/loader.py b/IPython/config/loader.py index a29e5d5..160739b 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -1,27 +1,8 @@ -"""A simple configuration system. +# encoding: utf-8 +"""A simple configuration system.""" -Inheritance diagram: - -.. inheritance-diagram:: IPython.config.loader - :parts: 3 - -Authors -------- -* Brian Granger -* Fernando Perez -* Min RK -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. import argparse import copy @@ -308,11 +289,8 @@ class ConfigLoader(object): """ def _log_default(self): - from IPython.config.application import Application - if Application.initialized(): - return Application.instance().log - else: - return logging.getLogger() + from IPython.utils.log import get_logger + return get_logger() def __init__(self, log=None): """A base class for config loaders. diff --git a/IPython/consoleapp.py b/IPython/consoleapp.py index 574df95..c7c5f0e 100644 --- a/IPython/consoleapp.py +++ b/IPython/consoleapp.py @@ -165,8 +165,6 @@ class IPythonConsoleApp(ConnectionFileMixin): if argv is None: argv = sys.argv[1:] self.kernel_argv = swallow_argv(argv, self.frontend_aliases, self.frontend_flags) - # kernel should inherit default config file from frontend - self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) def init_connection_file(self): """find the connection file, and load the info if found. @@ -289,6 +287,7 @@ class IPythonConsoleApp(ConnectionFileMixin): try: self.kernel_manager = self.kernel_manager_class( ip=self.ip, + session=self.session, transport=self.transport, shell_port=self.shell_port, iopub_port=self.iopub_port, @@ -326,6 +325,7 @@ class IPythonConsoleApp(ConnectionFileMixin): self.kernel_client = self.kernel_manager.client() else: self.kernel_client = self.kernel_client_class( + session=self.session, ip=self.ip, transport=self.transport, shell_port=self.shell_port, diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 71df7e1..ac61fec 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -80,6 +80,7 @@ from IPython.core.error import TryNext from IPython.core.inputsplitter import ESC_MAGIC from IPython.utils import generics from IPython.utils import io +from IPython.utils.decorators import undoc from IPython.utils.dir2 import dir2 from IPython.utils.process import arg_split from IPython.utils.py3compat import builtin_mod, string_types @@ -216,7 +217,7 @@ def penalize_magics_key(word): return word - +@undoc class Bunch(object): pass @@ -865,6 +866,7 @@ class IPCompleter(Completer): return argMatches def dict_key_matches(self, text): + "Match string keys in a dictionary, after e.g. 'foo[' " def get_keys(obj): # Only allow completion for known in-memory dict-like types if isinstance(obj, dict) or\ @@ -1010,9 +1012,6 @@ class IPCompleter(Completer): def complete(self, text=None, line_buffer=None, cursor_pos=None): """Find completions for the given text and line context. - This is called successively with state == 0, 1, 2, ... until it - returns None. The completion should begin with 'text'. - Note that both the text and the line_buffer are optional, but at least one of them must be given. diff --git a/IPython/core/display.py b/IPython/core/display.py index 61713cf..b53a85c 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -26,7 +26,13 @@ from IPython.core.formatters import _safe_get_formatter_method from IPython.utils.py3compat import (string_types, cast_bytes_py2, cast_unicode, unicode_type) from IPython.testing.skipdoctest import skip_doctest -from .displaypub import publish_display_data + +__all__ = ['display', 'display_pretty', 'display_html', 'display_markdown', +'display_svg', 'display_png', 'display_jpeg', 'display_latex', 'display_json', +'display_javascript', 'display_pdf', 'DisplayObject', 'TextDisplayObject', +'Pretty', 'HTML', 'Markdown', 'Math', 'Latex', 'SVG', 'JSON', 'Javascript', +'Image', 'clear_output', 'set_matplotlib_formats', 'set_matplotlib_close', +'publish_display_data'] #----------------------------------------------------------------------------- # utility functions @@ -78,6 +84,48 @@ def _display_mimetype(mimetype, objs, raw=False, metadata=None): # Main functions #----------------------------------------------------------------------------- +def publish_display_data(data, metadata=None, source=None): + """Publish data and metadata to all frontends. + + See the ``display_data`` message in the messaging documentation for + more details about this message type. + + The following MIME types are currently implemented: + + * text/plain + * text/html + * text/markdown + * text/latex + * application/json + * application/javascript + * image/png + * image/jpeg + * image/svg+xml + + Parameters + ---------- + data : dict + A dictionary having keys that are valid MIME types (like + 'text/plain' or 'image/svg+xml') and values that are the data for + that MIME type. The data itself must be a JSON'able data + structure. Minimally all data should have the 'text/plain' data, + which can be displayed by all frontends. If more than the plain + text is given, it is up to the frontend to decide which + representation to use. + metadata : dict + A dictionary for metadata related to the data. This can contain + arbitrary key, value pairs that frontends can use to interpret + the data. mime-type keys matching those in data can be used + to specify metadata about particular representations. + source : str, deprecated + Unused. + """ + from IPython.core.interactiveshell import InteractiveShell + InteractiveShell.instance().display_pub.publish( + data=data, + metadata=metadata, + ) + def display(*objs, **kwargs): """Display a Python object in all frontends. diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index 99c4a61..3a1985f 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -19,9 +19,11 @@ from __future__ import print_function from IPython.config.configurable import Configurable from IPython.utils import io -from IPython.utils.py3compat import string_types from IPython.utils.traitlets import List +# This used to be defined here - it is imported for backwards compatibility +from .display import publish_display_data + #----------------------------------------------------------------------------- # Main payload class #----------------------------------------------------------------------------- @@ -112,48 +114,3 @@ class CapturingDisplayPublisher(DisplayPublisher): # empty the list, *do not* reassign a new list del self.outputs[:] - - -def publish_display_data(data, metadata=None, source=None): - """Publish data and metadata to all frontends. - - See the ``display_data`` message in the messaging documentation for - more details about this message type. - - The following MIME types are currently implemented: - - * text/plain - * text/html - * text/markdown - * text/latex - * application/json - * application/javascript - * image/png - * image/jpeg - * image/svg+xml - - Parameters - ---------- - data : dict - A dictionary having keys that are valid MIME types (like - 'text/plain' or 'image/svg+xml') and values that are the data for - that MIME type. The data itself must be a JSON'able data - structure. Minimally all data should have the 'text/plain' data, - which can be displayed by all frontends. If more than the plain - text is given, it is up to the frontend to decide which - representation to use. - metadata : dict - A dictionary for metadata related to the data. This can contain - arbitrary key, value pairs that frontends can use to interpret - the data. mime-type keys matching those in data can be used - to specify metadata about particular representations. - source : str, deprecated - Unused. - """ - from IPython.core.interactiveshell import InteractiveShell - InteractiveShell.instance().display_pub.publish( - data=data, - metadata=metadata, - ) - - diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 427dafc..b78654e 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -736,12 +736,13 @@ class InteractiveShell(SingletonConfigurable): # stdlib venv may symlink sys.executable, so we can't use realpath. # but others can symlink *to* the venv Python, so we can't just use sys.executable. # So we just check every item in the symlink tree (generally <= 3) - p = sys.executable + p = os.path.normcase(sys.executable) paths = [p] while os.path.islink(p): - p = os.path.join(os.path.dirname(p), os.readlink(p)) + p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p))) paths.append(p) - if any(p.startswith(os.environ['VIRTUAL_ENV']) for p in paths): + p_venv = os.path.normcase(os.environ['VIRTUAL_ENV']) + if any(p.startswith(p_venv) for p in paths): # Running properly in the virtualenv, don't need to do anything return @@ -910,7 +911,8 @@ class InteractiveShell(SingletonConfigurable): try: main_mod = self._main_mod_cache[filename] except KeyError: - main_mod = self._main_mod_cache[filename] = types.ModuleType(modname, + main_mod = self._main_mod_cache[filename] = types.ModuleType( + py3compat.cast_bytes_py2(modname), doc="Module created for script run in IPython") else: main_mod.__dict__.clear() @@ -1735,7 +1737,7 @@ class InteractiveShell(SingletonConfigurable): This hook should be used sparingly, only in places which are not likely to be true IPython errors. """ - self.showtraceback((etype,value,tb),tb_offset=0) + self.showtraceback((etype, value, tb), tb_offset=0) def _get_exc_info(self, exc_tuple=None): """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc. @@ -1776,7 +1778,7 @@ class InteractiveShell(SingletonConfigurable): """ self.write_err("UsageError: %s" % exc) - def showtraceback(self,exc_tuple = None,filename=None,tb_offset=None, + def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None, exception_only=False): """Display the exception that just occurred. @@ -2918,10 +2920,9 @@ class InteractiveShell(SingletonConfigurable): False : successful execution. True : an error occurred. """ - # Set our own excepthook in case the user code tries to call it # directly, so that the IPython crash handler doesn't get triggered - old_excepthook,sys.excepthook = sys.excepthook, self.excepthook + old_excepthook, sys.excepthook = sys.excepthook, self.excepthook # we save the original sys.excepthook in the instance, in case config # code (such as magics) needs access to it. @@ -2939,8 +2940,8 @@ class InteractiveShell(SingletonConfigurable): self.showtraceback(exception_only=True) warn("To exit: use 'exit', 'quit', or Ctrl-D.", level=1) except self.custom_exceptions: - etype,value,tb = sys.exc_info() - self.CustomTB(etype,value,tb) + etype, value, tb = sys.exc_info() + self.CustomTB(etype, value, tb) except: self.showtraceback() else: @@ -3087,6 +3088,7 @@ class InteractiveShell(SingletonConfigurable): self.tempdirs.append(dirname) handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname) + os.close(handle) # On Windows, there can only be one open handle on a file self.tempfiles.append(filename) if data: diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py index e4fd1f0..e0b42cb 100644 --- a/IPython/core/magics/script.py +++ b/IPython/core/magics/script.py @@ -193,6 +193,8 @@ class ScriptMagics(Magics): else: raise + if not cell.endswith('\n'): + cell += '\n' cell = cell.encode('utf8', 'replace') if args.bg: self.bg_processes.append(p) diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index c08156b..347eb8b 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -6,6 +6,7 @@ #----------------------------------------------------------------------------- # stdlib +import io import os import sys import tempfile @@ -124,7 +125,7 @@ def test_history(): # Cross testing: check that magic %save can get previous session. testfilename = os.path.realpath(os.path.join(tmpdir, "test.py")) ip.magic("save " + testfilename + " ~1/1-3") - with py3compat.open(testfilename, encoding='utf-8') as testfile: + with io.open(testfilename, encoding='utf-8') as testfile: nt.assert_equal(testfile.read(), u"# coding: utf-8\n" + u"\n".join(hist)+u"\n") diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index d029206..de8d8d7 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -462,6 +462,21 @@ class InteractiveShellTestCase(unittest.TestCase): ip.run_cell("d = 1/2", shell_futures=True) self.assertEqual(ip.user_ns['d'], 0) + def test_mktempfile(self): + filename = ip.mktempfile() + # Check that we can open the file again on Windows + with open(filename, 'w') as f: + f.write('abc') + + filename = ip.mktempfile(data='blah') + with open(filename, 'r') as f: + self.assertEqual(f.read(), 'blah') + + def test_new_main_mod(self): + # Smoketest to check that this accepts a unicode module name + name = u'jiefmw' + mod = ip.new_main_mod(u'%s.py' % name, name) + self.assertEqual(mod.__name__, name) class TestSafeExecfileNonAsciiPath(unittest.TestCase): diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index 07c9ba5..c823d76 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -9,6 +9,7 @@ from IPython.testing import tools as tt from IPython.testing.decorators import onlyif_unicode_paths from IPython.utils.syspathcontext import prepended_to_syspath from IPython.utils.tempdir import TemporaryDirectory +from IPython.utils.py3compat import PY3 ip = get_ipython() @@ -147,3 +148,37 @@ class SyntaxErrorTest(unittest.TestCase): except ValueError: with tt.AssertPrints('QWERTY'): ip.showsyntaxerror() + + +class Python3ChainedExceptionsTest(unittest.TestCase): + DIRECT_CAUSE_ERROR_CODE = """ +try: + x = 1 + 2 + print(not_defined_here) +except Exception as e: + x += 55 + x - 1 + y = {} + raise KeyError('uh') from e + """ + + EXCEPTION_DURING_HANDLING_CODE = """ +try: + x = 1 + 2 + print(not_defined_here) +except Exception as e: + x += 55 + x - 1 + y = {} + raise KeyError('uh') + """ + + def test_direct_cause_error(self): + if PY3: + with tt.AssertPrints(["KeyError", "NameError", "direct cause"]): + ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE) + + def test_exception_during_handling_error(self): + if PY3: + with tt.AssertPrints(["KeyError", "NameError", "During handling"]): + ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index b1cc185..5fa0113 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -39,7 +39,7 @@ Give it a shot--you'll love it or you'll hate it. Verbose). -Installation instructions for ColorTB:: +Installation instructions for VerboseTB:: import sys,ultratb sys.excepthook = ultratb.VerboseTB() @@ -73,11 +73,11 @@ Inheritance diagram: """ #***************************************************************************** -# Copyright (C) 2001 Nathaniel Gray -# Copyright (C) 2001-2004 Fernando Perez +# Copyright (C) 2001 Nathaniel Gray +# Copyright (C) 2001-2004 Fernando Perez # -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. #***************************************************************************** from __future__ import unicode_literals @@ -95,14 +95,14 @@ import tokenize import traceback import types -try: # Python 2 +try: # Python 2 generate_tokens = tokenize.generate_tokens -except AttributeError: # Python 3 +except AttributeError: # Python 3 generate_tokens = tokenize.tokenize # For purposes of monkeypatching inspect to fix a bug in it. -from inspect import getsourcefile, getfile, getmodule,\ - ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode +from inspect import getsourcefile, getfile, getmodule, \ + ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode # IPython's own modules # Modified pdb which doesn't damage IPython's readline handling @@ -125,11 +125,11 @@ INDENT_SIZE = 8 # Default color scheme. This is used, for example, by the traceback # formatter. When running in an actual IPython instance, the user's rc.colors -# value is used, but havinga module global makes this functionality available +# value is used, but having a module global makes this functionality available # to users of ultratb who are NOT running inside ipython. DEFAULT_SCHEME = 'NoColor' -#--------------------------------------------------------------------------- +# --------------------------------------------------------------------------- # Code begins # Utility functions @@ -141,6 +141,7 @@ def inspect_error(): error('Internal Python error in the inspect module.\n' 'Below is the traceback from this internal error.\n') + # This function is a monkeypatch we apply to the Python inspect module. We have # now found when it's needed (see discussion on issue gh-1456), and we have a # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if @@ -212,7 +213,7 @@ def findsource(object): pmatch = pat.match # fperez - fix: sometimes, co_firstlineno can give a number larger than # the length of lines, which causes an error. Safeguard against that. - lnum = min(object.co_firstlineno,len(lines))-1 + lnum = min(object.co_firstlineno, len(lines)) - 1 while lnum > 0: if pmatch(lines[lnum]): break lnum -= 1 @@ -220,9 +221,11 @@ def findsource(object): return lines, lnum raise IOError('could not find code object') + # Monkeypatch inspect to apply our bugfix. def with_patch_inspect(f): """decorator for monkeypatching inspect.findsource""" + def wrapped(*args, **kwargs): save_findsource = inspect.findsource inspect.findsource = findsource @@ -230,8 +233,10 @@ def with_patch_inspect(f): return f(*args, **kwargs) finally: inspect.findsource = save_findsource + return wrapped + def fix_frame_records_filenames(records): """Try to fix the filenames in each record from inspect.getinnerframes(). @@ -253,11 +258,10 @@ def fix_frame_records_filenames(records): @with_patch_inspect -def _fixed_getinnerframes(etb, context=1,tb_offset=0): - LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5 - - records = fix_frame_records_filenames(inspect.getinnerframes(etb, context)) +def _fixed_getinnerframes(etb, context=1, tb_offset=0): + LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5 + records = fix_frame_records_filenames(inspect.getinnerframes(etb, context)) # If the error is at the console, don't build any context, since it would # otherwise produce 5 blank lines printed out (there is no file at the # console) @@ -272,9 +276,9 @@ def _fixed_getinnerframes(etb, context=1,tb_offset=0): aux = traceback.extract_tb(etb) assert len(records) == len(aux) for i, (file, lnum, _, _) in zip(range(len(records)), aux): - maybeStart = lnum-1 - context//2 - start = max(maybeStart, 0) - end = start + context + maybeStart = lnum - 1 - context // 2 + start = max(maybeStart, 0) + end = start + context lines = ulinecache.getlines(file)[start:end] buf = list(records[i]) buf[LNUM_POS] = lnum @@ -290,7 +294,8 @@ def _fixed_getinnerframes(etb, context=1,tb_offset=0): _parser = PyColorize.Parser() -def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None): + +def _format_traceback_lines(lnum, index, lines, Colors, lvals=None, scheme=None): numbers_width = INDENT_SIZE - 1 res = [] i = lnum - index @@ -315,7 +320,7 @@ def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None): # This is the line with the error pad = numbers_width - len(str(i)) if pad >= 3: - marker = '-'*(pad-3) + '-> ' + marker = '-' * (pad - 3) + '-> ' elif pad == 2: marker = '> ' elif pad == 1: @@ -323,12 +328,12 @@ def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,scheme=None): else: marker = '' num = marker + str(i) - line = '%s%s%s %s%s' %(Colors.linenoEm, num, - Colors.line, line, Colors.Normal) + line = '%s%s%s %s%s' % (Colors.linenoEm, num, + Colors.line, line, Colors.Normal) else: - num = '%*s' % (numbers_width,i) - line = '%s%s%s %s' %(Colors.lineno, num, - Colors.Normal, line) + num = '%*s' % (numbers_width, i) + line = '%s%s%s %s' % (Colors.lineno, num, + Colors.Normal, line) res.append(line) if lvals and i == lnum: @@ -389,16 +394,16 @@ class TBTools(object): ostream = property(_get_ostream, _set_ostream) - def set_colors(self,*args,**kw): + def set_colors(self, *args, **kw): """Shorthand access to the color table scheme selector method.""" # Set own color table - self.color_scheme_table.set_active_scheme(*args,**kw) + self.color_scheme_table.set_active_scheme(*args, **kw) # for convenience, set Colors to the active scheme self.Colors = self.color_scheme_table.active_colors # Also set colors of debugger - if hasattr(self,'pdb') and self.pdb is not None: - self.pdb.set_colors(*args,**kw) + if hasattr(self, 'pdb') and self.pdb is not None: + self.pdb.set_colors(*args, **kw) def color_toggle(self): """Toggle between the currently active color scheme and NoColor.""" @@ -453,7 +458,7 @@ class ListTB(TBTools): Because they are meant to be called without a full traceback (only a list), instances of this class can't call the interactive pdb debugger.""" - def __init__(self,color_scheme = 'NoColor', call_pdb=False, ostream=None): + def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None): TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, ostream=ostream) @@ -497,7 +502,7 @@ class ListTB(TBTools): elist = elist[tb_offset:] out_list.append('Traceback %s(most recent call last)%s:' % - (Colors.normalEm, Colors.Normal) + '\n') + (Colors.normalEm, Colors.Normal) + '\n') out_list.extend(self._format_list(elist)) # The exception info should be a single entry in the list. lines = ''.join(self._format_exception_only(etype, value)) @@ -510,7 +515,7 @@ class ListTB(TBTools): ## out_list.append(lines[-1]) # This means it was indenting everything but the last line by a little - # bit. I've disabled this for now, but if we see ugliness somewhre we + # bit. I've disabled this for now, but if we see ugliness somewhere we # can restore it. return out_list @@ -532,25 +537,24 @@ class ListTB(TBTools): list = [] for filename, lineno, name, line in extracted_list[:-1]: item = ' File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \ - (Colors.filename, filename, Colors.Normal, - Colors.lineno, lineno, Colors.Normal, - Colors.name, name, Colors.Normal) + (Colors.filename, filename, Colors.Normal, + Colors.lineno, lineno, Colors.Normal, + Colors.name, name, Colors.Normal) if line: item += ' %s\n' % line.strip() list.append(item) # Emphasize the last entry filename, lineno, name, line = extracted_list[-1] item = '%s File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \ - (Colors.normalEm, - Colors.filenameEm, filename, Colors.normalEm, - Colors.linenoEm, lineno, Colors.normalEm, - Colors.nameEm, name, Colors.normalEm, - Colors.Normal) + (Colors.normalEm, + Colors.filenameEm, filename, Colors.normalEm, + Colors.linenoEm, lineno, Colors.normalEm, + Colors.nameEm, name, Colors.normalEm, + Colors.Normal) if line: item += '%s %s%s\n' % (Colors.line, line.strip(), - Colors.Normal) + Colors.Normal) list.append(item) - #from pprint import pformat; print 'LISTTB', pformat(list) # dbg return list def _format_exception_only(self, etype, value): @@ -572,11 +576,10 @@ class ListTB(TBTools): stype = Colors.excName + etype.__name__ + Colors.Normal if value is None: # Not sure if this can still happen in Python 2.6 and above - list.append( py3compat.cast_unicode(stype) + '\n') + list.append(py3compat.cast_unicode(stype) + '\n') else: if issubclass(etype, SyntaxError): have_filedata = True - #print 'filename is',filename # dbg if not value.filename: value.filename = "" if value.lineno: lineno = value.lineno @@ -585,9 +588,9 @@ class ListTB(TBTools): lineno = 'unknown' textline = '' list.append('%s File %s"%s"%s, line %s%s%s\n' % \ - (Colors.normalEm, - Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm, - Colors.linenoEm, lineno, Colors.Normal )) + (Colors.normalEm, + Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm, + Colors.linenoEm, lineno, Colors.Normal )) if textline == '': textline = py3compat.cast_unicode(value.text, "utf-8") @@ -600,13 +603,13 @@ class ListTB(TBTools): Colors.Normal)) if value.offset is not None: s = ' ' - for c in textline[i:value.offset-1]: + for c in textline[i:value.offset - 1]: if c.isspace(): s += c else: s += ' ' list.append('%s%s^%s\n' % (Colors.caret, s, - Colors.Normal) ) + Colors.Normal)) try: s = value.msg @@ -636,7 +639,6 @@ class ListTB(TBTools): """ return ListTB.structured_traceback(self, etype, value, []) - def show_exception_only(self, etype, evalue): """Only print the exception type and message, without a traceback. @@ -659,6 +661,7 @@ class ListTB(TBTools): except: return '' % type(value).__name__ + #---------------------------------------------------------------------------- class VerboseTB(TBTools): """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead @@ -668,7 +671,7 @@ class VerboseTB(TBTools): traceback, to be used with alternate interpreters (because their own code would appear in the traceback).""" - def __init__(self,color_scheme = 'Linux', call_pdb=False, ostream=None, + def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None, tb_offset=0, long_header=False, include_vars=True, check_cache=None): """Specify traceback offset, headers and color scheme. @@ -691,126 +694,37 @@ class VerboseTB(TBTools): check_cache = linecache.checkcache self.check_cache = check_cache - def structured_traceback(self, etype, evalue, etb, tb_offset=None, - context=5): - """Return a nice text document describing the traceback.""" - - tb_offset = self.tb_offset if tb_offset is None else tb_offset - - # some locals - try: - etype = etype.__name__ - except AttributeError: - pass - Colors = self.Colors # just a shorthand + quicker name lookup - ColorsNormal = Colors.Normal # used a lot - col_scheme = self.color_scheme_table.active_scheme_name - indent = ' '*INDENT_SIZE - em_normal = '%s\n%s%s' % (Colors.valEm, indent,ColorsNormal) - undefined = '%sundefined%s' % (Colors.em, ColorsNormal) - exc = '%s%s%s' % (Colors.excName,etype,ColorsNormal) - - # some internal-use functions - def text_repr(value): - """Hopefully pretty robust repr equivalent.""" - # this is pretty horrible but should always return *something* - try: - return pydoc.text.repr(value) - except KeyboardInterrupt: - raise - except: - try: - return repr(value) - except KeyboardInterrupt: - raise - except: - try: - # all still in an except block so we catch - # getattr raising - name = getattr(value, '__name__', None) - if name: - # ick, recursion - return text_repr(name) - klass = getattr(value, '__class__', None) - if klass: - return '%s instance' % text_repr(klass) - except KeyboardInterrupt: - raise - except: - return 'UNRECOVERABLE REPR FAILURE' - def eqrepr(value, repr=text_repr): return '=%s' % repr(value) - def nullrepr(value, repr=text_repr): return '' - - # meat of the code begins - try: - etype = etype.__name__ - except AttributeError: - pass - - if self.long_header: - # Header with the exception type, python version, and date - pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable - date = time.ctime(time.time()) - - head = '%s%s%s\n%s%s%s\n%s' % (Colors.topline, '-'*75, ColorsNormal, - exc, ' '*(75-len(str(etype))-len(pyver)), - pyver, date.rjust(75) ) - head += "\nA problem occured executing Python code. Here is the sequence of function"\ - "\ncalls leading up to the error, with the most recent (innermost) call last." - else: - # Simplified header - head = '%s%s%s\n%s%s' % (Colors.topline, '-'*75, ColorsNormal,exc, - 'Traceback (most recent call last)'.\ - rjust(75 - len(str(etype)) ) ) + def format_records(self, records): + Colors = self.Colors # just a shorthand + quicker name lookup + ColorsNormal = Colors.Normal # used a lot + col_scheme = self.color_scheme_table.active_scheme_name + indent = ' ' * INDENT_SIZE + em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal) + undefined = '%sundefined%s' % (Colors.em, ColorsNormal) frames = [] - # Flush cache before calling inspect. This helps alleviate some of the - # problems with python 2.3's inspect.py. - ##self.check_cache() - # Drop topmost frames if requested - try: - # Try the default getinnerframes and Alex's: Alex's fixes some - # problems, but it generates empty tracebacks for console errors - # (5 blanks lines) where none should be returned. - #records = inspect.getinnerframes(etb, context)[tb_offset:] - #print 'python records:', records # dbg - records = _fixed_getinnerframes(etb, context, tb_offset) - #print 'alex records:', records # dbg - except: - - # FIXME: I've been getting many crash reports from python 2.3 - # users, traceable to inspect.py. If I can find a small test-case - # to reproduce this, I should either write a better workaround or - # file a bug report against inspect (if that's the real problem). - # So far, I haven't been able to find an isolated example to - # reproduce the problem. - inspect_error() - traceback.print_exc(file=self.ostream) - info('\nUnfortunately, your original traceback can not be constructed.\n') - return '' - # build some color string templates outside these nested loops - tpl_link = '%s%%s%s' % (Colors.filenameEm,ColorsNormal) - tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm, - ColorsNormal) - tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \ - (Colors.vName, Colors.valEm, ColorsNormal) - tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal) + tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal) + tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm, + ColorsNormal) + tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \ + (Colors.vName, Colors.valEm, ColorsNormal) + tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal) tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal, Colors.vName, ColorsNormal) - tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal) - tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal) - tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm,Colors.line, - ColorsNormal) + tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal) + + tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal) + tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm, Colors.line, + ColorsNormal) - # now, loop over all records printing context and info abspath = os.path.abspath for frame, file, lnum, func, lines, index in records: #print '*** record:',file,lnum,func,lines,index # dbg if not file: file = '?' - elif not(file.startswith(str("<")) and file.endswith(str(">"))): + elif not (file.startswith(str("<")) and file.endswith(str(">"))): # Guess that filenames like aren't real filenames, so - # don't call abspath on them. + # don't call abspath on them. try: file = abspath(file) except OSError: @@ -827,9 +741,9 @@ class VerboseTB(TBTools): # Decide whether to include variable details or not var_repr = self.include_vars and eqrepr or nullrepr try: - call = tpl_call % (func,inspect.formatargvalues(args, - varargs, varkw, - locals,formatvalue=var_repr)) + call = tpl_call % (func, inspect.formatargvalues(args, + varargs, varkw, + locals, formatvalue=var_repr)) except KeyError: # This happens in situations like errors inside generator # expressions, where local variables are listed in the @@ -848,12 +762,12 @@ class VerboseTB(TBTools): # will illustrate the error, if this exception catch is # disabled. call = tpl_call_fail % func - + # Don't attempt to tokenize binary files. if file.endswith(('.so', '.pyd', '.dll')): - frames.append('%s %s\n' % (link,call)) + frames.append('%s %s\n' % (link, call)) continue - elif file.endswith(('.pyc','.pyo')): + elif file.endswith(('.pyc', '.pyo')): # Look up the corresponding source file. file = openpy.source_from_cache(file) @@ -867,7 +781,7 @@ class VerboseTB(TBTools): try: names = [] name_cont = False - + for token_type, token, start, end, line in generate_tokens(linereader): # build composite names if token_type == tokenize.NAME and token not in keyword.kwlist: @@ -890,9 +804,11 @@ class VerboseTB(TBTools): name_cont = True elif token_type == tokenize.NEWLINE: break - - except (IndexError, UnicodeDecodeError): + + except (IndexError, UnicodeDecodeError, SyntaxError): # signals exit of tokenizer + # SyntaxError can occur if the file is not actually Python + # - see gh-6300 pass except tokenize.TokenError as msg: _m = ("An unexpected error occurred while tokenizing input\n" @@ -909,11 +825,11 @@ class VerboseTB(TBTools): lvals = [] if self.include_vars: for name_full in unique_names: - name_base = name_full.split('.',1)[0] + name_base = name_full.split('.', 1)[0] if name_base in frame.f_code.co_varnames: if name_base in locals: try: - value = repr(eval(name_full,locals)) + value = repr(eval(name_full, locals)) except: value = undefined else: @@ -922,69 +838,191 @@ class VerboseTB(TBTools): else: if name_base in frame.f_globals: try: - value = repr(eval(name_full,frame.f_globals)) + value = repr(eval(name_full, frame.f_globals)) except: value = undefined else: value = undefined name = tpl_global_var % name_full - lvals.append(tpl_name_val % (name,value)) + lvals.append(tpl_name_val % (name, value)) if lvals: - lvals = '%s%s' % (indent,em_normal.join(lvals)) + lvals = '%s%s' % (indent, em_normal.join(lvals)) else: lvals = '' - level = '%s %s\n' % (link,call) + level = '%s %s\n' % (link, call) if index is None: frames.append(level) else: - frames.append('%s%s' % (level,''.join( - _format_traceback_lines(lnum,index,lines,Colors,lvals, + frames.append('%s%s' % (level, ''.join( + _format_traceback_lines(lnum, index, lines, Colors, lvals, col_scheme)))) + return frames + + def prepare_chained_exception_message(self, cause): + direct_cause = "\nThe above exception was the direct cause of the following exception:\n" + exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n" + + if cause: + message = [[direct_cause]] + else: + message = [[exception_during_handling]] + return message + + def prepare_header(self, etype, long_version=False): + colors = self.Colors # just a shorthand + quicker name lookup + colorsnormal = colors.Normal # used a lot + exc = '%s%s%s' % (colors.excName, etype, colorsnormal) + if long_version: + # Header with the exception type, python version, and date + pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable + date = time.ctime(time.time()) + + head = '%s%s%s\n%s%s%s\n%s' % (colors.topline, '-' * 75, colorsnormal, + exc, ' ' * (75 - len(str(etype)) - len(pyver)), + pyver, date.rjust(75) ) + head += "\nA problem occurred executing Python code. Here is the sequence of function" \ + "\ncalls leading up to the error, with the most recent (innermost) call last." + else: + # Simplified header + head = '%s%s' % (exc, 'Traceback (most recent call last)'. \ + rjust(75 - len(str(etype))) ) + + return head + + def format_exception(self, etype, evalue): + colors = self.Colors # just a shorthand + quicker name lookup + colorsnormal = colors.Normal # used a lot + indent = ' ' * INDENT_SIZE # Get (safely) a string form of the exception info try: - etype_str,evalue_str = map(str,(etype,evalue)) + etype_str, evalue_str = map(str, (etype, evalue)) except: # User exception is improperly defined. - etype,evalue = str,sys.exc_info()[:2] - etype_str,evalue_str = map(str,(etype,evalue)) + etype, evalue = str, sys.exc_info()[:2] + etype_str, evalue_str = map(str, (etype, evalue)) # ... and format it - exception = ['%s%s%s: %s' % (Colors.excName, etype_str, - ColorsNormal, py3compat.cast_unicode(evalue_str))] + exception = ['%s%s%s: %s' % (colors.excName, etype_str, + colorsnormal, py3compat.cast_unicode(evalue_str))] + if (not py3compat.PY3) and type(evalue) is types.InstanceType: try: names = [w for w in dir(evalue) if isinstance(w, py3compat.string_types)] except: - # Every now and then, an object with funny inernals blows up + # Every now and then, an object with funny internals blows up # when dir() is called on it. We do the best we can to report # the problem and continue _m = '%sException reporting error (object with broken dir())%s:' - exception.append(_m % (Colors.excName,ColorsNormal)) - etype_str,evalue_str = map(str,sys.exc_info()[:2]) - exception.append('%s%s%s: %s' % (Colors.excName,etype_str, - ColorsNormal, py3compat.cast_unicode(evalue_str))) + exception.append(_m % (colors.excName, colorsnormal)) + etype_str, evalue_str = map(str, sys.exc_info()[:2]) + exception.append('%s%s%s: %s' % (colors.excName, etype_str, + colorsnormal, py3compat.cast_unicode(evalue_str))) names = [] for name in names: value = text_repr(getattr(evalue, name)) exception.append('\n%s%s = %s' % (indent, name, value)) - # vds: >> + return exception + + def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset): + # some locals + try: + etype = etype.__name__ + except AttributeError: + pass + + tb_offset = self.tb_offset if tb_offset is None else tb_offset + head = self.prepare_header(etype, self.long_header) + records = self.get_records(etb, number_of_lines_of_context, tb_offset) + + frames = self.format_records(records) + if records is None: + return "" + + formatted_exception = self.format_exception(etype, evalue) if records: - filepath, lnum = records[-1][1:3] - #print "file:", str(file), "linenb", str(lnum) # dbg - filepath = os.path.abspath(filepath) - ipinst = get_ipython() - if ipinst is not None: - ipinst.hooks.synchronize_with_editor(filepath, lnum, 0) - # vds: << - - # return all our info assembled as a single string - # return '%s\n\n%s\n%s' % (head,'\n'.join(frames),''.join(exception[0]) ) - return [head] + frames + [''.join(exception[0])] - - def debugger(self,force=False): + filepath, lnum = records[-1][1:3] + filepath = os.path.abspath(filepath) + ipinst = get_ipython() + if ipinst is not None: + ipinst.hooks.synchronize_with_editor(filepath, lnum, 0) + + return [[head] + frames + [''.join(formatted_exception[0])]] + + def get_records(self, etb, number_of_lines_of_context, tb_offset): + try: + # Try the default getinnerframes and Alex's: Alex's fixes some + # problems, but it generates empty tracebacks for console errors + # (5 blanks lines) where none should be returned. + return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset) + except: + # FIXME: I've been getting many crash reports from python 2.3 + # users, traceable to inspect.py. If I can find a small test-case + # to reproduce this, I should either write a better workaround or + # file a bug report against inspect (if that's the real problem). + # So far, I haven't been able to find an isolated example to + # reproduce the problem. + inspect_error() + traceback.print_exc(file=self.ostream) + info('\nUnfortunately, your original traceback can not be constructed.\n') + return None + + def get_parts_of_chained_exception(self, evalue): + def get_chained_exception(exception_value): + cause = getattr(exception_value, '__cause__', None) + if cause: + return cause + return getattr(exception_value, '__context__', None) + + chained_evalue = get_chained_exception(evalue) + + if chained_evalue: + return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__ + + def structured_traceback(self, etype, evalue, etb, tb_offset=None, + number_of_lines_of_context=5): + """Return a nice text document describing the traceback.""" + + formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context, + tb_offset) + + colors = self.Colors # just a shorthand + quicker name lookup + colorsnormal = colors.Normal # used a lot + head = '%s%s%s' % (colors.topline, '-' * 75, colorsnormal) + structured_traceback_parts = [head] + if py3compat.PY3: + chained_exceptions_tb_offset = 0 + lines_of_context = 3 + formatted_exceptions = formatted_exception + exception = self.get_parts_of_chained_exception(evalue) + if exception: + formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__) + etype, evalue, etb = exception + else: + evalue = None + while evalue: + formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context, + chained_exceptions_tb_offset) + exception = self.get_parts_of_chained_exception(evalue) + + if exception: + formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__) + etype, evalue, etb = exception + else: + evalue = None + + # we want to see exceptions in a reversed order: + # the first exception should be on top + for formatted_exception in reversed(formatted_exceptions): + structured_traceback_parts += formatted_exception + else: + structured_traceback_parts += formatted_exception[0] + + return structured_traceback_parts + + def debugger(self, force=False): """Call up the pdb debugger if desired, always clean up the tb reference. @@ -1014,7 +1052,7 @@ class VerboseTB(TBTools): with display_trap: self.pdb.reset() # Find the right frame so we don't pop up inside ipython itself - if hasattr(self,'tb') and self.tb is not None: + if hasattr(self, 'tb') and self.tb is not None: etb = self.tb else: etb = self.tb = sys.last_traceback @@ -1025,7 +1063,7 @@ class VerboseTB(TBTools): self.pdb.botframe = etb.tb_frame self.pdb.interaction(self.tb.tb_frame, self.tb) - if hasattr(self,'tb'): + if hasattr(self, 'tb'): del self.tb def handler(self, info=None): @@ -1050,6 +1088,7 @@ class VerboseTB(TBTools): except KeyboardInterrupt: print("\nKeyboardInterrupt") + #---------------------------------------------------------------------------- class FormattedTB(VerboseTB, ListTB): """Subclass ListTB but allow calling with a traceback. @@ -1069,7 +1108,7 @@ class FormattedTB(VerboseTB, ListTB): check_cache=None): # NEVER change the order of this list. Put new modes at the end: - self.valid_modes = ['Plain','Context','Verbose'] + self.valid_modes = ['Plain', 'Context', 'Verbose'] self.verbose_modes = self.valid_modes[1:3] VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb, @@ -1083,19 +1122,19 @@ class FormattedTB(VerboseTB, ListTB): # set_mode also sets the tb_join_char attribute self.set_mode(mode) - def _extract_tb(self,tb): + def _extract_tb(self, tb): if tb: return traceback.extract_tb(tb) else: return None - def structured_traceback(self, etype, value, tb, tb_offset=None, context=5): + def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5): tb_offset = self.tb_offset if tb_offset is None else tb_offset mode = self.mode if mode in self.verbose_modes: # Verbose modes need a full traceback return VerboseTB.structured_traceback( - self, etype, value, tb, tb_offset, context + self, etype, value, tb, tb_offset, number_of_lines_of_context ) else: # We must check the source cache because otherwise we can print @@ -1104,7 +1143,7 @@ class FormattedTB(VerboseTB, ListTB): # Now we can extract and format the exception elist = self._extract_tb(tb) return ListTB.structured_traceback( - self, etype, value, elist, tb_offset, context + self, etype, value, elist, tb_offset, number_of_lines_of_context ) def stb2text(self, stb): @@ -1112,18 +1151,18 @@ class FormattedTB(VerboseTB, ListTB): return self.tb_join_char.join(stb) - def set_mode(self,mode=None): + def set_mode(self, mode=None): """Switch to the desired mode. If mode is not specified, cycles through the available modes.""" if not mode: - new_idx = ( self.valid_modes.index(self.mode) + 1 ) % \ + new_idx = (self.valid_modes.index(self.mode) + 1 ) % \ len(self.valid_modes) self.mode = self.valid_modes[new_idx] elif mode not in self.valid_modes: - raise ValueError('Unrecognized mode in FormattedTB: <'+mode+'>\n' - 'Valid modes: '+str(self.valid_modes)) + raise ValueError('Unrecognized mode in FormattedTB: <' + mode + '>\n' + 'Valid modes: ' + str(self.valid_modes)) else: self.mode = mode # include variable details only in 'Verbose' mode @@ -1131,7 +1170,7 @@ class FormattedTB(VerboseTB, ListTB): # Set the join character for generating text tracebacks self.tb_join_char = self._join_chars[self.mode] - # some convenient shorcuts + # some convenient shortcuts def plain(self): self.set_mode(self.valid_modes[0]) @@ -1141,6 +1180,7 @@ class FormattedTB(VerboseTB, ListTB): def verbose(self): self.set_mode(self.valid_modes[2]) + #---------------------------------------------------------------------------- class AutoFormattedTB(FormattedTB): """A traceback printer which can be called on the fly. @@ -1156,8 +1196,8 @@ class AutoFormattedTB(FormattedTB): AutoTB() # or AutoTB(out=logfile) where logfile is an open file object """ - def __call__(self,etype=None,evalue=None,etb=None, - out=None,tb_offset=None): + def __call__(self, etype=None, evalue=None, etb=None, + out=None, tb_offset=None): """Print out a formatted exception traceback. Optional arguments: @@ -1167,7 +1207,6 @@ class AutoFormattedTB(FormattedTB): per-call basis (this overrides temporarily the instance's tb_offset given at initialization time. """ - if out is None: out = self.ostream out.flush() @@ -1182,33 +1221,36 @@ class AutoFormattedTB(FormattedTB): print("\nKeyboardInterrupt") def structured_traceback(self, etype=None, value=None, tb=None, - tb_offset=None, context=5): + tb_offset=None, number_of_lines_of_context=5): if etype is None: - etype,value,tb = sys.exc_info() + etype, value, tb = sys.exc_info() self.tb = tb return FormattedTB.structured_traceback( - self, etype, value, tb, tb_offset, context) + self, etype, value, tb, tb_offset, number_of_lines_of_context) + #--------------------------------------------------------------------------- # A simple class to preserve Nathan's original functionality. class ColorTB(FormattedTB): """Shorthand to initialize a FormattedTB in Linux colors mode.""" - def __init__(self,color_scheme='Linux',call_pdb=0): - FormattedTB.__init__(self,color_scheme=color_scheme, + + def __init__(self, color_scheme='Linux', call_pdb=0): + FormattedTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb) class SyntaxTB(ListTB): """Extension which holds some state: the last exception value""" - def __init__(self,color_scheme = 'NoColor'): - ListTB.__init__(self,color_scheme) + def __init__(self, color_scheme='NoColor'): + ListTB.__init__(self, color_scheme) self.last_syntax_error = None def __call__(self, etype, value, elist): self.last_syntax_error = value - ListTB.__call__(self,etype,value,elist) + + ListTB.__call__(self, etype, value, elist) def structured_traceback(self, etype, value, elist, tb_offset=None, context=5): @@ -1223,7 +1265,7 @@ class SyntaxTB(ListTB): if newtext: value.text = newtext return super(SyntaxTB, self).structured_traceback(etype, value, elist, - tb_offset=tb_offset, context=context) + tb_offset=tb_offset, context=context) def clear_err_state(self): """Return the current error state and clear it""" @@ -1236,7 +1278,46 @@ class SyntaxTB(ListTB): return ''.join(stb) +# some internal-use functions +def text_repr(value): + """Hopefully pretty robust repr equivalent.""" + # this is pretty horrible but should always return *something* + try: + return pydoc.text.repr(value) + except KeyboardInterrupt: + raise + except: + try: + return repr(value) + except KeyboardInterrupt: + raise + except: + try: + # all still in an except block so we catch + # getattr raising + name = getattr(value, '__name__', None) + if name: + # ick, recursion + return text_repr(name) + klass = getattr(value, '__class__', None) + if klass: + return '%s instance' % text_repr(klass) + except KeyboardInterrupt: + raise + except: + return 'UNRECOVERABLE REPR FAILURE' + + +def eqrepr(value, repr=text_repr): + return '=%s' % repr(value) + + +def nullrepr(value, repr=text_repr): + return '' + + #---------------------------------------------------------------------------- + # module testing (minimal) if __name__ == "__main__": def spam(c, d_e): diff --git a/IPython/external/mathjax.py b/IPython/external/mathjax.py index 532d9d3..a2a705c 100644 --- a/IPython/external/mathjax.py +++ b/IPython/external/mathjax.py @@ -136,7 +136,7 @@ def extract_zip(fd, dest): os.rename(os.path.join(parent, topdir), dest) -def install_mathjax(tag='v2.2', dest=default_dest, replace=False, file=None, extractor=extract_tar): +def install_mathjax(tag='2.4.0', dest=default_dest, replace=False, file=None, extractor=extract_tar): """Download and/or install MathJax for offline use. This will install mathjax to the nbextensions dir in your IPYTHONDIR. @@ -150,8 +150,8 @@ def install_mathjax(tag='v2.2', dest=default_dest, replace=False, file=None, ext Whether to remove and replace an existing install. dest : str [IPYTHONDIR/nbextensions/mathjax] Where to install mathjax - tag : str ['v2.2'] - Which tag to download. Default is 'v2.2', the current stable release, + tag : str ['2.4.0'] + Which tag to download. Default is '2.4.0', the current stable release, but alternatives include 'v1.1a' and 'master'. file : file like object [ defualt to content of https://github.com/mathjax/MathJax/tarball/#{tag}] File handle from which to untar/unzip/... mathjax diff --git a/IPython/external/pexpect/_pexpect.py b/IPython/external/pexpect/_pexpect.py index 942d19f..cace43b 100644 --- a/IPython/external/pexpect/_pexpect.py +++ b/IPython/external/pexpect/_pexpect.py @@ -80,6 +80,7 @@ try: import traceback import signal import codecs + import stat except ImportError: # pragma: no cover err = sys.exc_info()[1] raise ImportError(str(err) + ''' @@ -87,7 +88,7 @@ except ImportError: # pragma: no cover A critical module was not found. Probably this operating system does not support it. Pexpect is intended for UNIX-like operating systems.''') -__version__ = '3.2' +__version__ = '3.3' __revision__ = '' __all__ = ['ExceptionPexpect', 'EOF', 'TIMEOUT', 'spawn', 'spawnu', 'run', 'runu', 'which', 'split_command_line', '__version__', '__revision__'] @@ -284,6 +285,7 @@ class spawn(object): def _chr(c): return bytes([c]) linesep = os.linesep.encode('ascii') + crlf = '\r\n'.encode('ascii') @staticmethod def write_to_stdout(b): @@ -296,13 +298,14 @@ class spawn(object): allowed_string_types = (basestring,) # analysis:ignore _chr = staticmethod(chr) linesep = os.linesep + crlf = '\r\n' write_to_stdout = sys.stdout.write encoding = None def __init__(self, command, args=[], timeout=30, maxread=2000, searchwindowsize=None, logfile=None, cwd=None, env=None, - ignore_sighup=True): + ignore_sighup=True, echo=True): '''This is the constructor. The command parameter may be a string that includes a command and any arguments to the command. For example:: @@ -415,7 +418,16 @@ class spawn(object): signalstatus will store the signal value and exitstatus will be None. If you need more detail you can also read the self.status member which stores the status returned by os.waitpid. You can interpret this using - os.WIFEXITED/os.WEXITSTATUS or os.WIFSIGNALED/os.TERMSIG. ''' + os.WIFEXITED/os.WEXITSTATUS or os.WIFSIGNALED/os.TERMSIG. + + The echo attribute may be set to False to disable echoing of input. + As a pseudo-terminal, all input echoed by the "keyboard" (send() + or sendline()) will be repeated to output. For many cases, it is + not desirable to have echo enabled, and it may be later disabled + using setecho(False) followed by waitnoecho(). However, for some + platforms such as Solaris, this is not possible, and should be + disabled immediately on spawn. + ''' self.STDIN_FILENO = pty.STDIN_FILENO self.STDOUT_FILENO = pty.STDOUT_FILENO @@ -437,7 +449,7 @@ class spawn(object): self.status = None self.flag_eof = False self.pid = None - # the chile filedescriptor is initially closed + # the child file descriptor is initially closed self.child_fd = -1 self.timeout = timeout self.delimiter = EOF @@ -466,16 +478,30 @@ class spawn(object): self.closed = True self.cwd = cwd self.env = env + self.echo = echo self.ignore_sighup = ignore_sighup + _platform = sys.platform.lower() # This flags if we are running on irix - self.__irix_hack = (sys.platform.lower().find('irix') >= 0) + self.__irix_hack = _platform.startswith('irix') # Solaris uses internal __fork_pty(). All others use pty.fork(). - if ((sys.platform.lower().find('solaris') >= 0) - or (sys.platform.lower().find('sunos5') >= 0)): - self.use_native_pty_fork = False - else: - self.use_native_pty_fork = True - + self.use_native_pty_fork = not ( + _platform.startswith('solaris') or + _platform.startswith('sunos')) + # inherit EOF and INTR definitions from controlling process. + try: + from termios import VEOF, VINTR + fd = sys.__stdin__.fileno() + self._INTR = ord(termios.tcgetattr(fd)[6][VINTR]) + self._EOF = ord(termios.tcgetattr(fd)[6][VEOF]) + except (ImportError, OSError, IOError, termios.error): + # unless the controlling process is also not a terminal, + # such as cron(1). Fall-back to using CEOF and CINTR. + try: + from termios import CEOF, CINTR + (self._INTR, self._EOF) = (CINTR, CEOF) + except ImportError: + # ^C, ^D + (self._INTR, self._EOF) = (3, 4) # Support subclasses that do not use command or args. if command is None: self.command = None @@ -599,33 +625,39 @@ class spawn(object): if self.use_native_pty_fork: try: self.pid, self.child_fd = pty.fork() - except OSError: + except OSError: # pragma: no cover err = sys.exc_info()[1] raise ExceptionPexpect('pty.fork() failed: ' + str(err)) else: # Use internal __fork_pty self.pid, self.child_fd = self.__fork_pty() - if self.pid == 0: + # Some platforms must call setwinsize() and setecho() from the + # child process, and others from the master process. We do both, + # allowing IOError for either. + + if self.pid == pty.CHILD: # Child + self.child_fd = self.STDIN_FILENO + + # set default window size of 24 rows by 80 columns try: - # used by setwinsize() - self.child_fd = sys.stdout.fileno() self.setwinsize(24, 80) - # which exception, shouldnt' we catch explicitly .. ? - except: - # Some platforms do not like setwinsize (Cygwin). - # This will cause problem when running applications that - # are very picky about window size. - # This is a serious limitation, but not a show stopper. - pass + except IOError as err: + if err.args[0] not in (errno.EINVAL, errno.ENOTTY): + raise + + # disable echo if spawn argument echo was unset + if not self.echo: + try: + self.setecho(self.echo) + except (IOError, termios.error) as err: + if err.args[0] not in (errno.EINVAL, errno.ENOTTY): + raise + # Do not allow child to inherit open file descriptors from parent. max_fd = resource.getrlimit(resource.RLIMIT_NOFILE)[0] - for i in range(3, max_fd): - try: - os.close(i) - except OSError: - pass + os.closerange(3, max_fd) if self.ignore_sighup: signal.signal(signal.SIGHUP, signal.SIG_IGN) @@ -638,6 +670,13 @@ class spawn(object): os.execvpe(self.command, self.args, self.env) # Parent + try: + self.setwinsize(24, 80) + except IOError as err: + if err.args[0] not in (errno.EINVAL, errno.ENOTTY): + raise + + self.terminated = False self.closed = False @@ -660,19 +699,15 @@ class spawn(object): raise ExceptionPexpect("Could not open with os.openpty().") pid = os.fork() - if pid < 0: - raise ExceptionPexpect("Failed os.fork().") - elif pid == 0: + if pid == pty.CHILD: # Child. os.close(parent_fd) self.__pty_make_controlling_tty(child_fd) - os.dup2(child_fd, 0) - os.dup2(child_fd, 1) - os.dup2(child_fd, 2) + os.dup2(child_fd, self.STDIN_FILENO) + os.dup2(child_fd, self.STDOUT_FILENO) + os.dup2(child_fd, self.STDERR_FILENO) - if child_fd > 2: - os.close(child_fd) else: # Parent. os.close(child_fd) @@ -686,44 +721,36 @@ class spawn(object): child_name = os.ttyname(tty_fd) - # Disconnect from controlling tty. Harmless if not already connected. + # Disconnect from controlling tty, if any. Raises OSError of ENXIO + # if there was no controlling tty to begin with, such as when + # executed by a cron(1) job. try: fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY) - if fd >= 0: - os.close(fd) - # which exception, shouldnt' we catch explicitly .. ? - except: - # Already disconnected. This happens if running inside cron. - pass + os.close(fd) + except OSError as err: + if err.errno != errno.ENXIO: + raise os.setsid() - # Verify we are disconnected from controlling tty - # by attempting to open it again. + # Verify we are disconnected from controlling tty by attempting to open + # it again. We expect that OSError of ENXIO should always be raised. try: fd = os.open("/dev/tty", os.O_RDWR | os.O_NOCTTY) - if fd >= 0: - os.close(fd) - raise ExceptionPexpect('Failed to disconnect from ' + - 'controlling tty. It is still possible to open /dev/tty.') - # which exception, shouldnt' we catch explicitly .. ? - except: - # Good! We are disconnected from a controlling tty. - pass + os.close(fd) + raise ExceptionPexpect("OSError of errno.ENXIO should be raised.") + except OSError as err: + if err.errno != errno.ENXIO: + raise # Verify we can open child pty. fd = os.open(child_name, os.O_RDWR) - if fd < 0: - raise ExceptionPexpect("Could not open child pty, " + child_name) - else: - os.close(fd) + os.close(fd) # Verify we now have a controlling tty. fd = os.open("/dev/tty", os.O_WRONLY) - if fd < 0: - raise ExceptionPexpect("Could not open controlling tty, /dev/tty") - else: - os.close(fd) + os.close(fd) + def fileno(self): '''This returns the file descriptor of the pty for the child. @@ -757,7 +784,12 @@ class spawn(object): def isatty(self): '''This returns True if the file descriptor is open and connected to a - tty(-like) device, else False. ''' + tty(-like) device, else False. + + On SVR4-style platforms implementing streams, such as SunOS and HP-UX, + the child pty may not appear as a terminal device. This means + methods such as setecho(), setwinsize(), getwinsize() may raise an + IOError. ''' return os.isatty(self.child_fd) @@ -794,12 +826,20 @@ class spawn(object): def getecho(self): '''This returns the terminal echo mode. This returns True if echo is on or False if echo is off. Child applications that are expecting you - to enter a password often set ECHO False. See waitnoecho(). ''' + to enter a password often set ECHO False. See waitnoecho(). - attr = termios.tcgetattr(self.child_fd) - if attr[3] & termios.ECHO: - return True - return False + Not supported on platforms where ``isatty()`` returns False. ''' + + try: + attr = termios.tcgetattr(self.child_fd) + except termios.error as err: + errmsg = 'getecho() may not be called on this platform' + if err.args[0] == errno.EINVAL: + raise IOError(err.args[0], '%s: %s.' % (err.args[1], errmsg)) + raise + + self.echo = bool(attr[3] & termios.ECHO) + return self.echo def setecho(self, state): '''This sets the terminal echo mode on or off. Note that anything the @@ -829,18 +869,35 @@ class spawn(object): p.expect(['1234']) p.expect(['abcd']) p.expect(['wxyz']) + + + Not supported on platforms where ``isatty()`` returns False. ''' - self.child_fd - attr = termios.tcgetattr(self.child_fd) + errmsg = 'setecho() may not be called on this platform' + + try: + attr = termios.tcgetattr(self.child_fd) + except termios.error as err: + if err.args[0] == errno.EINVAL: + raise IOError(err.args[0], '%s: %s.' % (err.args[1], errmsg)) + raise + if state: attr[3] = attr[3] | termios.ECHO else: attr[3] = attr[3] & ~termios.ECHO - # I tried TCSADRAIN and TCSAFLUSH, but - # these were inconsistent and blocked on some platforms. - # TCSADRAIN would probably be ideal if it worked. - termios.tcsetattr(self.child_fd, termios.TCSANOW, attr) + + try: + # I tried TCSADRAIN and TCSAFLUSH, but these were inconsistent and + # blocked on some platforms. TCSADRAIN would probably be ideal. + termios.tcsetattr(self.child_fd, termios.TCSANOW, attr) + except IOError as err: + if err.args[0] == errno.EINVAL: + raise IOError(err.args[0], '%s: %s.' % (err.args[1], errmsg)) + raise + + self.echo = state def _log(self, s, direction): if self.logfile is not None: @@ -913,12 +970,14 @@ class spawn(object): if self.child_fd in r: try: s = os.read(self.child_fd, size) - except OSError: - # Linux does this - self.flag_eof = True - raise EOF('End Of File (EOF). Exception style platform.') + except OSError as err: + if err.args[0] == errno.EIO: + # Linux-style EOF + self.flag_eof = True + raise EOF('End Of File (EOF). Exception style platform.') + raise if s == b'': - # BSD style + # BSD-style EOF self.flag_eof = True raise EOF('End Of File (EOF). Empty string style platform.') @@ -926,7 +985,7 @@ class spawn(object): self._log(s, 'read') return s - raise ExceptionPexpect('Reached an unexpected state.') + raise ExceptionPexpect('Reached an unexpected state.') # pragma: no cover def read(self, size=-1): '''This reads at most "size" bytes from the file (less if the read hits @@ -972,9 +1031,9 @@ class spawn(object): if size == 0: return self.string_type() # delimiter default is EOF - index = self.expect([b'\r\n', self.delimiter]) + index = self.expect([self.crlf, self.delimiter]) if index == 0: - return self.before + b'\r\n' + return self.before + self.crlf else: return self.before @@ -1075,40 +1134,14 @@ class spawn(object): It is the responsibility of the caller to ensure the eof is sent at the beginning of a line. ''' - ### Hmmm... how do I send an EOF? - ###C if ((m = write(pty, *buf, p - *buf)) < 0) - ###C return (errno == EWOULDBLOCK) ? n : -1; - #fd = sys.stdin.fileno() - #old = termios.tcgetattr(fd) # remember current state - #attr = termios.tcgetattr(fd) - #attr[3] = attr[3] | termios.ICANON # ICANON must be set to see EOF - #try: # use try/finally to ensure state gets restored - # termios.tcsetattr(fd, termios.TCSADRAIN, attr) - # if hasattr(termios, 'CEOF'): - # os.write(self.child_fd, '%c' % termios.CEOF) - # else: - # # Silly platform does not define CEOF so assume CTRL-D - # os.write(self.child_fd, '%c' % 4) - #finally: # restore state - # termios.tcsetattr(fd, termios.TCSADRAIN, old) - if hasattr(termios, 'VEOF'): - char = ord(termios.tcgetattr(self.child_fd)[6][termios.VEOF]) - else: - # platform does not define VEOF so assume CTRL-D - char = 4 - self.send(self._chr(char)) + self.send(self._chr(self._EOF)) def sendintr(self): '''This sends a SIGINT to the child. It does not require the SIGINT to be the first character on a line. ''' - if hasattr(termios, 'VINTR'): - char = ord(termios.tcgetattr(self.child_fd)[6][termios.VINTR]) - else: - # platform does not define VINTR so assume CTRL-C - char = 3 - self.send(self._chr(char)) + self.send(self._chr(self._INTR)) def eof(self): @@ -1181,7 +1214,7 @@ class spawn(object): self.exitstatus = None self.signalstatus = os.WTERMSIG(status) self.terminated = True - elif os.WIFSTOPPED(status): + elif os.WIFSTOPPED(status): # pragma: no cover # You can't call wait() on a child process in the stopped state. raise ExceptionPexpect('Called wait() on a stopped child ' + 'process. This is not supported. Is some other ' + @@ -1201,7 +1234,7 @@ class spawn(object): if self.flag_eof: # This is for Linux, which requires the blocking form - # of waitpid to # get status of a defunct process. + # of waitpid to get the status of a defunct process. # This is super-lame. The flag_eof would have been set # in read_nonblocking(), so this should be safe. waitpid_options = 0 @@ -1229,7 +1262,7 @@ class spawn(object): try: ### os.WNOHANG) # Solaris! pid, status = os.waitpid(self.pid, waitpid_options) - except OSError as e: + except OSError as e: # pragma: no cover # This should never happen... if e.errno == errno.ECHILD: raise ExceptionPexpect('isalive() encountered condition ' + @@ -1643,10 +1676,14 @@ class spawn(object): if self.child_fd in r: try: data = self.__interact_read(self.child_fd) - except OSError as e: - # The subprocess may have closed before we get to reading it - if e.errno != errno.EIO: - raise + except OSError as err: + if err.args[0] == errno.EIO: + # Linux-style EOF + break + raise + if data == b'': + # BSD-style EOF + break if output_filter: data = output_filter(data) if self.logfile is not None: @@ -1695,7 +1732,7 @@ class spawn(object): ############################################################################## # The following methods are no longer supported or allowed. - def setmaxread(self, maxread): + def setmaxread(self, maxread): # pragma: no cover '''This method is no longer supported or allowed. I don't like getters and setters without a good reason. ''' @@ -1704,7 +1741,7 @@ class spawn(object): 'or allowed. Just assign a value to the ' + 'maxread member variable.') - def setlog(self, fileobject): + def setlog(self, fileobject): # pragma: no cover '''This method is no longer supported or allowed. ''' @@ -1732,11 +1769,13 @@ class spawnu(spawn): allowed_string_types = (str, ) _chr = staticmethod(chr) linesep = os.linesep + crlf = '\r\n' else: string_type = unicode allowed_string_types = (unicode, ) _chr = staticmethod(unichr) linesep = os.linesep.decode('ascii') + crlf = '\r\n'.decode('ascii') # This can handle unicode in both Python 2 and 3 write_to_stdout = sys.stdout.write @@ -1959,16 +1998,56 @@ class searcher_re(object): return best_index -def which(filename): +def is_executable_file(path): + """Checks that path is an executable regular file (or a symlink to a file). + + This is roughly ``os.path isfile(path) and os.access(path, os.X_OK)``, but + on some platforms :func:`os.access` gives us the wrong answer, so this + checks permission bits directly. + """ + # follow symlinks, + fpath = os.path.realpath(path) + # return False for non-files (directories, fifo, etc.) + if not os.path.isfile(fpath): + return False + + # On Solaris, etc., "If the process has appropriate privileges, an + # implementation may indicate success for X_OK even if none of the + # execute file permission bits are set." + # + # For this reason, it is necessary to explicitly check st_mode + + # get file mode using os.stat, and check if `other', + # that is anybody, may read and execute. + mode = os.stat(fpath).st_mode + if mode & stat.S_IROTH and mode & stat.S_IXOTH: + return True + + # get current user's group ids, and check if `group', + # when matching ours, may read and execute. + user_gids = os.getgroups() + [os.getgid()] + if (os.stat(fpath).st_gid in user_gids and + mode & stat.S_IRGRP and mode & stat.S_IXGRP): + return True + + # finally, if file owner matches our effective userid, + # check if `user', may read and execute. + user_gids = os.getgroups() + [os.getgid()] + if (os.stat(fpath).st_uid == os.geteuid() and + mode & stat.S_IRUSR and mode & stat.S_IXUSR): + return True + + return False + +def which(filename): '''This takes a given filename; tries to find it in the environment path; then checks if it is executable. This returns the full path to the filename if found and executable. Otherwise this returns None.''' # Special case where filename contains an explicit path. - if os.path.dirname(filename) != '': - if os.access(filename, os.X_OK): - return filename + if os.path.dirname(filename) != '' and is_executable_file(filename): + return filename if 'PATH' not in os.environ or os.environ['PATH'] == '': p = os.defpath else: @@ -1976,7 +2055,7 @@ def which(filename): pathlist = p.split(os.pathsep) for path in pathlist: ff = os.path.join(path, filename) - if os.access(ff, os.X_OK): + if is_executable_file(ff): return ff return None @@ -2041,4 +2120,4 @@ def split_command_line(command_line): arg_list.append(arg) return arg_list -# vi:set sr et ts=4 sw=4 ft=python : +# vim: set shiftround expandtab tabstop=4 shiftwidth=4 ft=python autoindent : diff --git a/IPython/html/README.md b/IPython/html/README.md index 2f3f6f3..e354f71 100644 --- a/IPython/html/README.md +++ b/IPython/html/README.md @@ -13,19 +13,18 @@ Developers of the IPython Notebook will need to install the following tools: We are moving to a model where our JavaScript dependencies are managed using [bower](http://bower.io/). These packages are installed in `static/components` -and committed into our git repo. Our dependencies are described in the file +and committed into a separate git repo [ipython/ipython-components](ipython/ipython-components). +Our dependencies are described in the file `static/components/bower.json`. To update our bower packages, run `fab update` in this directory. -Because CodeMirror does not use proper semantic versioning for its GitHub tags, -we maintain our own fork of CodeMirror that is used with bower. This fork should -track the upstream CodeMirror exactly; the only difference is that we are adding -semantic versioned tags to our repo. - ## less If you edit our `.less` files you will need to run the less compiler to build -our minified css files. This can be done by running `fab css` from this directory. +our minified css files. This can be done by running `fab css` from this directory, +or `python setup.py css` from the root of the repository. +If you are working frequently with `.less` files please consider installing git hooks that +rebuild the css files and corresponding maps in `${RepoRoot}/git-hooks/install-hooks.sh`. ## JavaScript Documentation diff --git a/IPython/html/base/handlers.py b/IPython/html/base/handlers.py index 56772f9..07bc059 100644 --- a/IPython/html/base/handlers.py +++ b/IPython/html/base/handlers.py @@ -1,21 +1,7 @@ -"""Base Tornado handlers for the notebook. - -Authors: - -* Brian Granger -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +"""Base Tornado handlers for the notebook server.""" +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. import functools import json @@ -41,7 +27,7 @@ except ImportError: from IPython.config import Application from IPython.utils.path import filefind from IPython.utils.py3compat import string_types -from IPython.html.utils import is_hidden +from IPython.html.utils import is_hidden, url_path_join, url_escape #----------------------------------------------------------------------------- # Top-level handlers @@ -53,6 +39,10 @@ class AuthenticatedHandler(web.RequestHandler): def set_default_headers(self): headers = self.settings.get('headers', {}) + + if "X-Frame-Options" not in headers: + headers["X-Frame-Options"] = "SAMEORIGIN" + for header_name,value in headers.items() : try: self.set_header(header_name, value) @@ -137,6 +127,10 @@ class IPythonHandler(AuthenticatedHandler): @property def base_url(self): return self.settings.get('base_url', '/') + + @property + def ws_url(self): + return self.settings.get('websocket_url', '') #--------------------------------------------------------------- # Manager objects @@ -147,8 +141,8 @@ class IPythonHandler(AuthenticatedHandler): return self.settings['kernel_manager'] @property - def notebook_manager(self): - return self.settings['notebook_manager'] + def contents_manager(self): + return self.settings['contents_manager'] @property def cluster_manager(self): @@ -162,9 +156,47 @@ class IPythonHandler(AuthenticatedHandler): def kernel_spec_manager(self): return self.settings['kernel_spec_manager'] + #--------------------------------------------------------------- + # CORS + #--------------------------------------------------------------- + @property - def project_dir(self): - return self.notebook_manager.notebook_dir + def allow_origin(self): + """Normal Access-Control-Allow-Origin""" + return self.settings.get('allow_origin', '') + + @property + def allow_origin_pat(self): + """Regular expression version of allow_origin""" + return self.settings.get('allow_origin_pat', None) + + @property + def allow_credentials(self): + """Whether to set Access-Control-Allow-Credentials""" + return self.settings.get('allow_credentials', False) + + def set_default_headers(self): + """Add CORS headers, if defined""" + super(IPythonHandler, self).set_default_headers() + if self.allow_origin: + self.set_header("Access-Control-Allow-Origin", self.allow_origin) + elif self.allow_origin_pat: + origin = self.get_origin() + if origin and self.allow_origin_pat.match(origin): + self.set_header("Access-Control-Allow-Origin", origin) + if self.allow_credentials: + self.set_header("Access-Control-Allow-Credentials", 'true') + + def get_origin(self): + # Handle WebSocket Origin naming convention differences + # The difference between version 8 and 13 is that in 8 the + # client sends a "Sec-Websocket-Origin" header and in 13 it's + # simply "Origin". + if "Origin" in self.request.headers: + origin = self.request.headers.get("Origin") + else: + origin = self.request.headers.get("Sec-Websocket-Origin", None) + return origin #--------------------------------------------------------------- # template rendering @@ -183,6 +215,7 @@ class IPythonHandler(AuthenticatedHandler): def template_namespace(self): return dict( base_url=self.base_url, + ws_url=self.ws_url, logged_in=self.logged_in, login_available=self.login_available, static_url=self.static_url, @@ -202,12 +235,13 @@ class IPythonHandler(AuthenticatedHandler): raise web.HTTPError(400, u'Invalid JSON in body of request') return model - def get_error_html(self, status_code, **kwargs): + def write_error(self, status_code, **kwargs): """render custom error pages""" - exception = kwargs.get('exception') + exc_info = kwargs.get('exc_info') message = '' status_message = responses.get(status_code, 'Unknown HTTP Error') - if exception: + if exc_info: + exception = exc_info[1] # get the custom message, if defined try: message = exception.log_message % exception.args @@ -227,13 +261,16 @@ class IPythonHandler(AuthenticatedHandler): exception=exception, ) + self.set_header('Content-Type', 'text/html') # render the template try: html = self.render_template('%s.html' % status_code, **ns) except TemplateNotFound: self.log.debug("No template for %d", status_code) html = self.render_template('error.html', **ns) - return html + + self.write(html) + class Template404(IPythonHandler): @@ -372,6 +409,37 @@ class TrailingSlashHandler(web.RequestHandler): def get(self): self.redirect(self.request.uri.rstrip('/')) + +class FilesRedirectHandler(IPythonHandler): + """Handler for redirecting relative URLs to the /files/ handler""" + def get(self, path=''): + cm = self.contents_manager + if cm.path_exists(path): + # it's a *directory*, redirect to /tree + url = url_path_join(self.base_url, 'tree', path) + else: + orig_path = path + # otherwise, redirect to /files + parts = path.split('/') + path = '/'.join(parts[:-1]) + name = parts[-1] + + if not cm.file_exists(name=name, path=path) and 'files' in parts: + # redirect without files/ iff it would 404 + # this preserves pre-2.0-style 'files/' links + self.log.warn("Deprecated files/ URL: %s", orig_path) + parts.remove('files') + path = '/'.join(parts[:-1]) + + if not cm.file_exists(name=name, path=path): + raise web.HTTPError(404) + + url = url_path_join(self.base_url, 'files', path, name) + url = url_escape(url) + self.log.debug("Redirecting %s to %s", self.request.path, url) + self.redirect(url) + + #----------------------------------------------------------------------------- # URL pattern fragments for re-use #----------------------------------------------------------------------------- @@ -379,6 +447,8 @@ class TrailingSlashHandler(web.RequestHandler): path_regex = r"(?P(?:/.*)*)" notebook_name_regex = r"(?P[^/]+\.ipynb)" notebook_path_regex = "%s/%s" % (path_regex, notebook_name_regex) +file_name_regex = r"(?P[^/]+)" +file_path_regex = "%s/%s" % (path_regex, file_name_regex) #----------------------------------------------------------------------------- # URL to handler mappings diff --git a/IPython/html/base/zmqhandlers.py b/IPython/html/base/zmqhandlers.py index 8999b26..d69155b 100644 --- a/IPython/html/base/zmqhandlers.py +++ b/IPython/html/base/zmqhandlers.py @@ -15,6 +15,9 @@ try: except ImportError: from Cookie import SimpleCookie # Py 2 import logging + +import tornado +from tornado import ioloop from tornado import web from tornado import websocket @@ -26,29 +29,36 @@ from .handlers import IPythonHandler class ZMQStreamHandler(websocket.WebSocketHandler): - - def same_origin(self): - """Check to see that origin and host match in the headers.""" - - # The difference between version 8 and 13 is that in 8 the - # client sends a "Sec-Websocket-Origin" header and in 13 it's - # simply "Origin". - if self.request.headers.get("Sec-WebSocket-Version") in ("7", "8"): - origin_header = self.request.headers.get("Sec-Websocket-Origin") - else: - origin_header = self.request.headers.get("Origin") + + def check_origin(self, origin): + """Check Origin == Host or Access-Control-Allow-Origin. + + Tornado >= 4 calls this method automatically, raising 403 if it returns False. + We call it explicitly in `open` on Tornado < 4. + """ + if self.allow_origin == '*': + return True host = self.request.headers.get("Host") # If no header is provided, assume we can't verify origin - if(origin_header is None or host is None): + if(origin is None or host is None): + return False + + host_origin = "{0}://{1}".format(self.request.protocol, host) + + # OK if origin matches host + if origin == host_origin: + return True + + # Check CORS headers + if self.allow_origin: + return self.allow_origin == origin + elif self.allow_origin_pat: + return bool(self.allow_origin_pat.match(origin)) + else: + # No CORS headers deny the request return False - - parsed_origin = urlparse(origin_header) - origin = parsed_origin.netloc - - # Check to see that origin matches host directly, including ports - return origin == host def clear_cookie(self, *args, **kwargs): """meaningless for websockets""" @@ -94,19 +104,41 @@ class ZMQStreamHandler(websocket.WebSocketHandler): """ return True +# ping interval for keeping websockets alive (30 seconds) +WS_PING_INTERVAL = 30000 class AuthenticatedZMQStreamHandler(ZMQStreamHandler, IPythonHandler): + ping_callback = None + + def set_default_headers(self): + """Undo the set_default_headers in IPythonHandler + + which doesn't make sense for websockets + """ + pass def open(self, kernel_id): self.kernel_id = cast_unicode(kernel_id, 'ascii') # Check to see that origin matches host directly, including ports - if not self.same_origin(): - self.log.warn("Cross Origin WebSocket Attempt.") - raise web.HTTPError(404) + # Tornado 4 already does CORS checking + if tornado.version_info[0] < 4: + if not self.check_origin(self.get_origin()): + self.log.warn("Cross Origin WebSocket Attempt from %s", self.get_origin()) + raise web.HTTPError(403) self.session = Session(config=self.config) self.save_on_message = self.on_message self.on_message = self.on_first_message + self.ping_callback = ioloop.PeriodicCallback(self.send_ping, WS_PING_INTERVAL) + self.ping_callback.start() + + def send_ping(self): + """send a ping to keep the websocket alive""" + if self.stream.closed() and self.ping_callback is not None: + self.ping_callback.stop() + return + + self.ping(b'') def _inject_cookie_message(self, msg): """Inject the first message, which is the document cookie, diff --git a/IPython/html/fabfile.py b/IPython/html/fabfile.py index db2d48a..2583d8d 100644 --- a/IPython/html/fabfile.py +++ b/IPython/html/fabfile.py @@ -8,31 +8,65 @@ from subprocess import check_output pjoin = os.path.join static_dir = 'static' -components_dir = os.path.join(static_dir, 'components') +components_dir = pjoin(static_dir, 'components') +here = os.path.dirname(__file__) -min_less_version = '1.4.0' -max_less_version = '1.5.0' # exclusive +min_less_version = '1.7.0' +max_less_version = '1.8.0' # exclusive -def css(minify=True, verbose=False): +def _need_css_update(): + """Does less need to run?""" + + static_path = pjoin(here, static_dir) + css_targets = [ + pjoin(static_path, 'style', '%s.min.css' % name) + for name in ('style', 'ipython') + ] + css_maps = [t + '.map' for t in css_targets] + targets = css_targets + css_maps + if not all(os.path.exists(t) for t in targets): + # some generated files don't exist + return True + earliest_target = sorted(os.stat(t).st_mtime for t in targets)[0] + + # check if any .less files are newer than the generated targets + for (dirpath, dirnames, filenames) in os.walk(static_path): + for f in filenames: + if f.endswith('.less'): + path = pjoin(static_path, dirpath, f) + timestamp = os.stat(path).st_mtime + if timestamp > earliest_target: + return True + + return False + +def css(minify=False, verbose=False, force=False): """generate the css from less files""" + minify = _to_bool(minify) + verbose = _to_bool(verbose) + force = _to_bool(force) + # minify implies force because it's not the default behavior + if not force and not minify and not _need_css_update(): + print("css up-to-date") + return + for name in ('style', 'ipython'): source = pjoin('style', "%s.less" % name) target = pjoin('style', "%s.min.css" % name) - _compile_less(source, target, minify, verbose) + sourcemap = pjoin('style', "%s.min.css.map" % name) + _compile_less(source, target, sourcemap, minify, verbose) def _to_bool(b): if not b in ['True', 'False', True, False]: abort('boolean expected, got: %s' % b) return (b in ['True', True]) -def _compile_less(source, target, minify=True, verbose=False): +def _compile_less(source, target, sourcemap, minify=True, verbose=False): """Compile a less file by source and target relative to static_dir""" - minify = _to_bool(minify) - verbose = _to_bool(verbose) min_flag = '-x' if minify is True else '' ver_flag = '--verbose' if verbose is True else '' - # pin less to 1.4 + # pin less to version number from above try: out = check_output(['lessc', '--version']) except OSError as err: @@ -45,6 +79,7 @@ def _compile_less(source, target, minify=True, verbose=False): if V(less_version) >= V(max_less_version): raise ValueError("lessc too new: %s >= %s. Use `$ npm install lesscss@X.Y.Z` to install a specific version of less" % (less_version, max_less_version)) + static_path = pjoin(here, static_dir) with lcd(static_dir): - local('lessc {min_flag} {ver_flag} {source} {target}'.format(**locals())) + local('lessc {min_flag} {ver_flag} --source-map={sourcemap} --source-map-basepath={static_path} --source-map-rootpath="../" {source} {target}'.format(**locals())) diff --git a/IPython/html/nbconvert/handlers.py b/IPython/html/nbconvert/handlers.py index fb97f5f..f6e1094 100644 --- a/IPython/html/nbconvert/handlers.py +++ b/IPython/html/nbconvert/handlers.py @@ -1,10 +1,18 @@ +"""Tornado handlers for nbconvert.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + import io import os import zipfile from tornado import web -from ..base.handlers import IPythonHandler, notebook_path_regex +from ..base.handlers import ( + IPythonHandler, FilesRedirectHandler, + notebook_path_regex, path_regex, +) from IPython.nbformat.current import to_notebook_json from IPython.utils.py3compat import cast_bytes @@ -73,7 +81,7 @@ class NbconvertFileHandler(IPythonHandler): exporter = get_exporter(format, config=self.config, log=self.log) path = path.strip('/') - model = self.notebook_manager.get_notebook(name=name, path=path) + model = self.contents_manager.get_model(name=name, path=path) self.set_header('Last-Modified', model['last_modified']) @@ -123,6 +131,7 @@ class NbconvertPostHandler(IPythonHandler): self.finish(output) + #----------------------------------------------------------------------------- # URL to handler mappings #----------------------------------------------------------------------------- @@ -134,4 +143,5 @@ default_handlers = [ (r"/nbconvert/%s%s" % (_format_regex, notebook_path_regex), NbconvertFileHandler), (r"/nbconvert/%s" % _format_regex, NbconvertPostHandler), + (r"/nbconvert/html%s" % path_regex, FilesRedirectHandler), ] diff --git a/IPython/html/nbconvert/tests/test_nbconvert_handlers.py b/IPython/html/nbconvert/tests/test_nbconvert_handlers.py index 6916f1f..ea44217 100644 --- a/IPython/html/nbconvert/tests/test_nbconvert_handlers.py +++ b/IPython/html/nbconvert/tests/test_nbconvert_handlers.py @@ -106,7 +106,7 @@ class APITest(NotebookTestBase): @onlyif_cmds_exist('pandoc') def test_from_post(self): - nbmodel_url = url_path_join(self.base_url(), 'api/notebooks/foo/testnb.ipynb') + nbmodel_url = url_path_join(self.base_url(), 'api/contents/foo/testnb.ipynb') nbmodel = requests.get(nbmodel_url).json() r = self.nbconvert_api.from_post(format='html', nbmodel=nbmodel) @@ -121,7 +121,7 @@ class APITest(NotebookTestBase): @onlyif_cmds_exist('pandoc') def test_from_post_zip(self): - nbmodel_url = url_path_join(self.base_url(), 'api/notebooks/foo/testnb.ipynb') + nbmodel_url = url_path_join(self.base_url(), 'api/contents/foo/testnb.ipynb') nbmodel = requests.get(nbmodel_url).json() r = self.nbconvert_api.from_post(format='latex', nbmodel=nbmodel) diff --git a/IPython/html/notebook/handlers.py b/IPython/html/notebook/handlers.py index 5db20cc..31e5ee7 100644 --- a/IPython/html/notebook/handlers.py +++ b/IPython/html/notebook/handlers.py @@ -1,31 +1,17 @@ -"""Tornado handlers for the live notebook view. +"""Tornado handlers for the live notebook view.""" -Authors: - -* Brian Granger -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. import os from tornado import web HTTPError = web.HTTPError -from ..base.handlers import IPythonHandler, notebook_path_regex, path_regex -from ..utils import url_path_join, url_escape - -#----------------------------------------------------------------------------- -# Handlers -#----------------------------------------------------------------------------- +from ..base.handlers import ( + IPythonHandler, FilesRedirectHandler, + notebook_path_regex, path_regex, +) +from ..utils import url_escape class NotebookHandler(IPythonHandler): @@ -35,17 +21,16 @@ class NotebookHandler(IPythonHandler): """get renders the notebook template if a name is given, or redirects to the '/files/' handler if the name is not given.""" path = path.strip('/') - nbm = self.notebook_manager + cm = self.contents_manager if name is None: raise web.HTTPError(500, "This shouldn't be accessible: %s" % self.request.uri) # a .ipynb filename was given - if not nbm.notebook_exists(name, path): + if not cm.file_exists(name, path): raise web.HTTPError(404, u'Notebook does not exist: %s/%s' % (path, name)) name = url_escape(name) path = url_escape(path) self.write(self.render_template('notebook.html', - project=self.project_dir, notebook_path=path, notebook_name=name, kill_kernel=False, @@ -53,30 +38,6 @@ class NotebookHandler(IPythonHandler): ) ) -class NotebookRedirectHandler(IPythonHandler): - def get(self, path=''): - nbm = self.notebook_manager - if nbm.path_exists(path): - # it's a *directory*, redirect to /tree - url = url_path_join(self.base_url, 'tree', path) - else: - # otherwise, redirect to /files - if '/files/' in path: - # redirect without files/ iff it would 404 - # this preserves pre-2.0-style 'files/' links - # FIXME: this is hardcoded based on notebook_path, - # but so is the files handler itself, - # so it should work until both are cleaned up. - parts = path.split('/') - files_path = os.path.join(nbm.notebook_dir, *parts) - if not os.path.exists(files_path): - self.log.warn("Deprecated files/ URL: %s", path) - path = path.replace('/files/', '/', 1) - - url = url_path_join(self.base_url, 'files', path) - url = url_escape(url) - self.log.debug("Redirecting %s to %s", self.request.path, url) - self.redirect(url) #----------------------------------------------------------------------------- # URL to handler mappings @@ -85,6 +46,6 @@ class NotebookRedirectHandler(IPythonHandler): default_handlers = [ (r"/notebooks%s" % notebook_path_regex, NotebookHandler), - (r"/notebooks%s" % path_regex, NotebookRedirectHandler), + (r"/notebooks%s" % path_regex, FilesRedirectHandler), ] diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py index 9c555d6..a1c24c6 100644 --- a/IPython/html/notebookapp.py +++ b/IPython/html/notebookapp.py @@ -6,12 +6,14 @@ from __future__ import print_function +import base64 import errno import io import json import logging import os import random +import re import select import signal import socket @@ -53,8 +55,8 @@ from IPython.html import DEFAULT_STATIC_FILES_PATH from .base.handlers import Template404 from .log import log_request from .services.kernels.kernelmanager import MappingKernelManager -from .services.notebooks.nbmanager import NotebookManager -from .services.notebooks.filenbmanager import FileNotebookManager +from .services.contents.manager import ContentsManager +from .services.contents.filemanager import FileContentsManager from .services.clusters.clustermanager import ClusterManager from .services.sessions.sessionmanager import SessionManager @@ -72,6 +74,7 @@ from IPython.kernel.zmq.session import default_secure, Session from IPython.nbformat.sign import NotebookNotary from IPython.utils.importstring import import_item from IPython.utils import submodule +from IPython.utils.process import check_pid from IPython.utils.traitlets import ( Dict, Unicode, Integer, List, Bool, Bytes, Instance, DottedObjectName, TraitError, @@ -118,19 +121,19 @@ def load_handlers(name): class NotebookWebApplication(web.Application): - def __init__(self, ipython_app, kernel_manager, notebook_manager, + def __init__(self, ipython_app, kernel_manager, contents_manager, cluster_manager, session_manager, kernel_spec_manager, log, base_url, settings_overrides, jinja_env_options): settings = self.init_settings( - ipython_app, kernel_manager, notebook_manager, cluster_manager, + ipython_app, kernel_manager, contents_manager, cluster_manager, session_manager, kernel_spec_manager, log, base_url, settings_overrides, jinja_env_options) handlers = self.init_handlers(settings) super(NotebookWebApplication, self).__init__(handlers, **settings) - def init_settings(self, ipython_app, kernel_manager, notebook_manager, + def init_settings(self, ipython_app, kernel_manager, contents_manager, cluster_manager, session_manager, kernel_spec_manager, log, base_url, settings_overrides, jinja_env_options=None): @@ -162,13 +165,14 @@ class NotebookWebApplication(web.Application): # managers kernel_manager=kernel_manager, - notebook_manager=notebook_manager, + contents_manager=contents_manager, cluster_manager=cluster_manager, session_manager=session_manager, kernel_spec_manager=kernel_spec_manager, # IPython stuff nbextensions_path = ipython_app.nbextensions_path, + websocket_url=ipython_app.websocket_url, mathjax_url=ipython_app.mathjax_url, config=ipython_app.config, jinja2_env=env, @@ -189,18 +193,20 @@ class NotebookWebApplication(web.Application): handlers.extend(load_handlers('nbconvert.handlers')) handlers.extend(load_handlers('kernelspecs.handlers')) handlers.extend(load_handlers('services.kernels.handlers')) - handlers.extend(load_handlers('services.notebooks.handlers')) + handlers.extend(load_handlers('services.contents.handlers')) handlers.extend(load_handlers('services.clusters.handlers')) handlers.extend(load_handlers('services.sessions.handlers')) handlers.extend(load_handlers('services.nbconvert.handlers')) handlers.extend(load_handlers('services.kernelspecs.handlers')) # FIXME: /files/ should be handled by the Contents service when it exists - nbm = settings['notebook_manager'] - if hasattr(nbm, 'notebook_dir'): - handlers.extend([ - (r"/files/(.*)", AuthenticatedFileHandler, {'path' : nbm.notebook_dir}), + cm = settings['contents_manager'] + if hasattr(cm, 'root_dir'): + handlers.append( + (r"/files/(.*)", AuthenticatedFileHandler, {'path' : cm.root_dir}), + ) + handlers.append( (r"/nbextensions/(.*)", FileFindHandler, {'path' : settings['nbextensions_path']}), - ]) + ) # prepend base_url onto the patterns that we match new_handlers = [] for handler in handlers: @@ -260,9 +266,9 @@ flags['no-mathjax']=( ) # Add notebook manager flags -flags.update(boolean_flag('script', 'FileNotebookManager.save_script', - 'Auto-save a .py script everytime the .ipynb notebook is saved', - 'Do not auto-save .py scripts for every notebook')) +flags.update(boolean_flag('script', 'FileContentsManager.save_script', + 'DEPRECATED, IGNORED', + 'DEPRECATED, IGNORED')) aliases = dict(base_aliases) @@ -298,7 +304,7 @@ class NotebookApp(BaseIPythonApplication): classes = [ KernelManager, ProfileDir, Session, MappingKernelManager, - NotebookManager, FileNotebookManager, NotebookNotary, + ContentsManager, FileContentsManager, NotebookNotary, ] flags = Dict(flags) aliases = Dict(aliases) @@ -333,8 +339,34 @@ class NotebookApp(BaseIPythonApplication): self.file_to_run = base self.notebook_dir = path - # Network related information. - + # Network related information + + allow_origin = Unicode('', config=True, + help="""Set the Access-Control-Allow-Origin header + + Use '*' to allow any origin to access your server. + + Takes precedence over allow_origin_pat. + """ + ) + + allow_origin_pat = Unicode('', config=True, + help="""Use a regular expression for the Access-Control-Allow-Origin header + + Requests from an origin matching the expression will get replies with: + + Access-Control-Allow-Origin: origin + + where `origin` is the origin of the request. + + Ignored if allow_origin is set. + """ + ) + + allow_credentials = Bool(False, config=True, + help="Set the Access-Control-Allow-Credentials: true header" + ) + ip = Unicode('localhost', config=True, help="The IP address the notebook server will listen on." ) @@ -357,6 +389,14 @@ class NotebookApp(BaseIPythonApplication): help="""The full path to a private key file for usage with SSL/TLS.""" ) + cookie_secret_file = Unicode(config=True, + help="""The file where the cookie secret is stored.""" + ) + def _cookie_secret_file_default(self): + if self.profile_dir is None: + return '' + return os.path.join(self.profile_dir.security_dir, 'notebook_cookie_secret') + cookie_secret = Bytes(b'', config=True, help="""The random bytes used to secure cookies. By default this is a new random number every time you start the Notebook. @@ -367,7 +407,26 @@ class NotebookApp(BaseIPythonApplication): """ ) def _cookie_secret_default(self): - return os.urandom(1024) + if os.path.exists(self.cookie_secret_file): + with io.open(self.cookie_secret_file, 'rb') as f: + return f.read() + else: + secret = base64.encodestring(os.urandom(1024)) + self._write_cookie_secret_file(secret) + return secret + + def _write_cookie_secret_file(self, secret): + """write my secret to my secret_file""" + self.log.info("Writing notebook server cookie secret to %s", self.cookie_secret_file) + with io.open(self.cookie_secret_file, 'wb') as f: + f.write(secret) + try: + os.chmod(self.cookie_secret_file, 0o600) + except OSError: + self.log.warn( + "Could not set permissions on %s", + self.cookie_secret_file + ) password = Unicode(u'', config=True, help="""Hashed password to use for web authentication. @@ -456,6 +515,13 @@ class NotebookApp(BaseIPythonApplication): def _nbextensions_path_default(self): return [os.path.join(get_ipython_dir(), 'nbextensions')] + websocket_url = Unicode("", config=True, + help="""The base URL for websockets, + if it differs from the HTTP server (hint: it almost certainly doesn't). + + Should be in the form of an HTTP origin: ws[s]://hostname[:port] + """ + ) mathjax_url = Unicode("", config=True, help="""The url for MathJax.js.""" ) @@ -482,13 +548,7 @@ class NotebookApp(BaseIPythonApplication): return url # no local mathjax, serve from CDN - if self.certfile: - # HTTPS: load from Rackspace CDN, because SSL certificate requires it - host = u"https://c328740.ssl.cf1.rackcdn.com" - else: - host = u"http://cdn.mathjax.org" - - url = host + u"/mathjax/latest/MathJax.js" + url = u"https://cdn.mathjax.org/mathjax/latest/MathJax.js" self.log.info("Using MathJax from CDN: %s", url) return url @@ -499,7 +559,7 @@ class NotebookApp(BaseIPythonApplication): else: self.log.info("Using MathJax: %s", new) - notebook_manager_class = DottedObjectName('IPython.html.services.notebooks.filenbmanager.FileNotebookManager', + contents_manager_class = DottedObjectName('IPython.html.services.contents.filemanager.FileContentsManager', config=True, help='The notebook manager class to use.' ) @@ -563,7 +623,7 @@ class NotebookApp(BaseIPythonApplication): raise TraitError("No such notebook dir: %r" % new) # setting App.notebook_dir implies setting notebook and kernel dirs as well - self.config.FileNotebookManager.notebook_dir = new + self.config.FileContentsManager.root_dir = new self.config.MappingKernelManager.root_dir = new @@ -589,11 +649,8 @@ class NotebookApp(BaseIPythonApplication): def init_kernel_argv(self): """construct the kernel arguments""" - self.kernel_argv = [] - # Kernel should inherit default config file from frontend - self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) # Kernel should get *absolute* path to profile directory - self.kernel_argv.extend(["--profile-dir", self.profile_dir.location]) + self.kernel_argv = ["--profile-dir", self.profile_dir.location] def init_configurables(self): # force Session default to be secure @@ -603,10 +660,12 @@ class NotebookApp(BaseIPythonApplication): parent=self, log=self.log, kernel_argv=self.kernel_argv, connection_dir = self.profile_dir.security_dir, ) - kls = import_item(self.notebook_manager_class) - self.notebook_manager = kls(parent=self, log=self.log) + kls = import_item(self.contents_manager_class) + self.contents_manager = kls(parent=self, log=self.log) kls = import_item(self.session_manager_class) - self.session_manager = kls(parent=self, log=self.log) + self.session_manager = kls(parent=self, log=self.log, + kernel_manager=self.kernel_manager, + contents_manager=self.contents_manager) kls = import_item(self.cluster_manager_class) self.cluster_manager = kls(parent=self, log=self.log) self.cluster_manager.update_profiles() @@ -625,8 +684,13 @@ class NotebookApp(BaseIPythonApplication): def init_webapp(self): """initialize tornado webapp and httpserver""" + self.webapp_settings['allow_origin'] = self.allow_origin + if self.allow_origin_pat: + self.webapp_settings['allow_origin_pat'] = re.compile(self.allow_origin_pat) + self.webapp_settings['allow_credentials'] = self.allow_credentials + self.web_app = NotebookWebApplication( - self, self.kernel_manager, self.notebook_manager, + self, self.kernel_manager, self.contents_manager, self.cluster_manager, self.session_manager, self.kernel_spec_manager, self.log, self.base_url, self.webapp_settings, self.jinja_environment_options @@ -717,8 +781,6 @@ class NotebookApp(BaseIPythonApplication): This doesn't work on Windows. """ - # FIXME: remove this delay when pyzmq dependency is >= 2.1.11 - time.sleep(0.1) info = self.log.info info('interrupted') print(self.notebook_info()) @@ -778,7 +840,7 @@ class NotebookApp(BaseIPythonApplication): def notebook_info(self): "Return the current working directory and the server url information" - info = self.notebook_manager.info_string() + "\n" + info = self.contents_manager.info_string() + "\n" info += "%d active kernels \n" % len(self.kernel_manager._kernels) return info + "The IPython Notebook is running at: %s" % self.display_url @@ -790,6 +852,7 @@ class NotebookApp(BaseIPythonApplication): 'secure': bool(self.certfile), 'base_url': self.base_url, 'notebook_dir': os.path.abspath(self.notebook_dir), + 'pid': os.getpid() } def write_server_info_file(self): @@ -863,8 +926,17 @@ def list_running_servers(profile='default'): for file in os.listdir(pd.security_dir): if file.startswith('nbserver-'): with io.open(os.path.join(pd.security_dir, file), encoding='utf-8') as f: - yield json.load(f) + info = json.load(f) + # Simple check whether that process is really still running + if check_pid(info['pid']): + yield info + else: + # If the process has died, try to delete its info file + try: + os.unlink(file) + except OSError: + pass # TODO: This should warn or log or something #----------------------------------------------------------------------------- # Main entry point #----------------------------------------------------------------------------- diff --git a/IPython/html/services/clusters/clustermanager.py b/IPython/html/services/clusters/clustermanager.py index 66fb529..fecde70 100644 --- a/IPython/html/services/clusters/clustermanager.py +++ b/IPython/html/services/clusters/clustermanager.py @@ -21,7 +21,6 @@ from zmq.eventloop import ioloop from IPython.config.configurable import LoggingConfigurable from IPython.utils.traitlets import Dict, Instance, CFloat -from IPython.parallel.apps.ipclusterapp import IPClusterStart from IPython.core.profileapp import list_profiles_in from IPython.core.profiledir import ProfileDir from IPython.utils import py3compat @@ -33,17 +32,6 @@ from IPython.utils.path import get_ipython_dir #----------------------------------------------------------------------------- -class DummyIPClusterStart(IPClusterStart): - """Dummy subclass to skip init steps that conflict with global app. - - Instantiating and initializing this class should result in fully configured - launchers, but no other side effects or state. - """ - - def init_signal(self): - pass - def reinit_logging(self): - pass class ClusterManager(LoggingConfigurable): @@ -59,6 +47,20 @@ class ClusterManager(LoggingConfigurable): return IOLoop.instance() def build_launchers(self, profile_dir): + from IPython.parallel.apps.ipclusterapp import IPClusterStart + + class DummyIPClusterStart(IPClusterStart): + """Dummy subclass to skip init steps that conflict with global app. + + Instantiating and initializing this class should result in fully configured + launchers, but no other side effects or state. + """ + + def init_signal(self): + pass + def reinit_logging(self): + pass + starter = DummyIPClusterStart(log=self.log) starter.initialize(['--profile-dir', profile_dir]) cl = starter.controller_launcher diff --git a/IPython/html/services/notebooks/__init__.py b/IPython/html/services/contents/__init__.py similarity index 100% rename from IPython/html/services/notebooks/__init__.py rename to IPython/html/services/contents/__init__.py diff --git a/IPython/html/services/contents/filemanager.py b/IPython/html/services/contents/filemanager.py new file mode 100644 index 0000000..59da309 --- /dev/null +++ b/IPython/html/services/contents/filemanager.py @@ -0,0 +1,531 @@ +"""A contents manager that uses the local file system for storage.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import base64 +import io +import os +import glob +import shutil + +from tornado import web + +from .manager import ContentsManager +from IPython.nbformat import current +from IPython.utils.path import ensure_dir_exists +from IPython.utils.traitlets import Unicode, Bool, TraitError +from IPython.utils.py3compat import getcwd +from IPython.utils import tz +from IPython.html.utils import is_hidden, to_os_path, url_path_join + + +class FileContentsManager(ContentsManager): + + root_dir = Unicode(getcwd(), config=True) + + save_script = Bool(False, config=True, help='DEPRECATED, IGNORED') + def _save_script_changed(self): + self.log.warn(""" + Automatically saving notebooks as scripts has been removed. + Use `ipython nbconvert --to python [notebook]` instead. + """) + + def _root_dir_changed(self, name, old, new): + """Do a bit of validation of the root_dir.""" + if not os.path.isabs(new): + # If we receive a non-absolute path, make it absolute. + self.root_dir = os.path.abspath(new) + return + if not os.path.isdir(new): + raise TraitError("%r is not a directory" % new) + + checkpoint_dir = Unicode('.ipynb_checkpoints', config=True, + help="""The directory name in which to keep file checkpoints + + This is a path relative to the file's own directory. + + By default, it is .ipynb_checkpoints + """ + ) + + def _copy(self, src, dest): + """copy src to dest + + like shutil.copy2, but log errors in copystat + """ + shutil.copyfile(src, dest) + try: + shutil.copystat(src, dest) + except OSError as e: + self.log.debug("copystat on %s failed", dest, exc_info=True) + + def _get_os_path(self, name=None, path=''): + """Given a filename and API path, return its file system + path. + + Parameters + ---------- + name : string + A filename + path : string + The relative API path to the named file. + + Returns + ------- + path : string + API path to be evaluated relative to root_dir. + """ + if name is not None: + path = url_path_join(path, name) + return to_os_path(path, self.root_dir) + + def path_exists(self, path): + """Does the API-style path refer to an extant directory? + + API-style wrapper for os.path.isdir + + Parameters + ---------- + path : string + The path to check. This is an API path (`/` separated, + relative to root_dir). + + Returns + ------- + exists : bool + Whether the path is indeed a directory. + """ + path = path.strip('/') + os_path = self._get_os_path(path=path) + return os.path.isdir(os_path) + + def is_hidden(self, path): + """Does the API style path correspond to a hidden directory or file? + + Parameters + ---------- + path : string + The path to check. This is an API path (`/` separated, + relative to root_dir). + + Returns + ------- + exists : bool + Whether the path is hidden. + + """ + path = path.strip('/') + os_path = self._get_os_path(path=path) + return is_hidden(os_path, self.root_dir) + + def file_exists(self, name, path=''): + """Returns True if the file exists, else returns False. + + API-style wrapper for os.path.isfile + + Parameters + ---------- + name : string + The name of the file you are checking. + path : string + The relative path to the file's directory (with '/' as separator) + + Returns + ------- + exists : bool + Whether the file exists. + """ + path = path.strip('/') + nbpath = self._get_os_path(name, path=path) + return os.path.isfile(nbpath) + + def exists(self, name=None, path=''): + """Returns True if the path [and name] exists, else returns False. + + API-style wrapper for os.path.exists + + Parameters + ---------- + name : string + The name of the file you are checking. + path : string + The relative path to the file's directory (with '/' as separator) + + Returns + ------- + exists : bool + Whether the target exists. + """ + path = path.strip('/') + os_path = self._get_os_path(name, path=path) + return os.path.exists(os_path) + + def _base_model(self, name, path=''): + """Build the common base of a contents model""" + os_path = self._get_os_path(name, path) + info = os.stat(os_path) + last_modified = tz.utcfromtimestamp(info.st_mtime) + created = tz.utcfromtimestamp(info.st_ctime) + # Create the base model. + model = {} + model['name'] = name + model['path'] = path + model['last_modified'] = last_modified + model['created'] = created + model['content'] = None + model['format'] = None + return model + + def _dir_model(self, name, path='', content=True): + """Build a model for a directory + + if content is requested, will include a listing of the directory + """ + os_path = self._get_os_path(name, path) + + four_o_four = u'directory does not exist: %r' % os_path + + if not os.path.isdir(os_path): + raise web.HTTPError(404, four_o_four) + elif is_hidden(os_path, self.root_dir): + self.log.info("Refusing to serve hidden directory %r, via 404 Error", + os_path + ) + raise web.HTTPError(404, four_o_four) + + if name is None: + if '/' in path: + path, name = path.rsplit('/', 1) + else: + name = '' + model = self._base_model(name, path) + model['type'] = 'directory' + dir_path = u'{}/{}'.format(path, name) + if content: + model['content'] = contents = [] + for os_path in glob.glob(self._get_os_path('*', dir_path)): + name = os.path.basename(os_path) + if self.should_list(name) and not is_hidden(os_path, self.root_dir): + contents.append(self.get_model(name=name, path=dir_path, content=False)) + + model['format'] = 'json' + + return model + + def _file_model(self, name, path='', content=True): + """Build a model for a file + + if content is requested, include the file contents. + UTF-8 text files will be unicode, binary files will be base64-encoded. + """ + model = self._base_model(name, path) + model['type'] = 'file' + if content: + os_path = self._get_os_path(name, path) + with io.open(os_path, 'rb') as f: + bcontent = f.read() + try: + model['content'] = bcontent.decode('utf8') + except UnicodeError as e: + model['content'] = base64.encodestring(bcontent).decode('ascii') + model['format'] = 'base64' + else: + model['format'] = 'text' + return model + + + def _notebook_model(self, name, path='', content=True): + """Build a notebook model + + if content is requested, the notebook content will be populated + as a JSON structure (not double-serialized) + """ + model = self._base_model(name, path) + model['type'] = 'notebook' + if content: + os_path = self._get_os_path(name, path) + with io.open(os_path, 'r', encoding='utf-8') as f: + try: + nb = current.read(f, u'json') + except Exception as e: + raise web.HTTPError(400, u"Unreadable Notebook: %s %s" % (os_path, e)) + self.mark_trusted_cells(nb, name, path) + model['content'] = nb + model['format'] = 'json' + return model + + def get_model(self, name, path='', content=True): + """ Takes a path and name for an entity and returns its model + + Parameters + ---------- + name : str + the name of the target + path : str + the API path that describes the relative path for the target + + Returns + ------- + model : dict + the contents model. If content=True, returns the contents + of the file or directory as well. + """ + path = path.strip('/') + + if not self.exists(name=name, path=path): + raise web.HTTPError(404, u'No such file or directory: %s/%s' % (path, name)) + + os_path = self._get_os_path(name, path) + if os.path.isdir(os_path): + model = self._dir_model(name, path, content) + elif name.endswith('.ipynb'): + model = self._notebook_model(name, path, content) + else: + model = self._file_model(name, path, content) + return model + + def _save_notebook(self, os_path, model, name='', path=''): + """save a notebook file""" + # Save the notebook file + nb = current.to_notebook_json(model['content']) + + self.check_and_sign(nb, name, path) + + if 'name' in nb['metadata']: + nb['metadata']['name'] = u'' + + with io.open(os_path, 'w', encoding='utf-8') as f: + current.write(nb, f, u'json') + + def _save_file(self, os_path, model, name='', path=''): + """save a non-notebook file""" + fmt = model.get('format', None) + if fmt not in {'text', 'base64'}: + raise web.HTTPError(400, "Must specify format of file contents as 'text' or 'base64'") + try: + content = model['content'] + if fmt == 'text': + bcontent = content.encode('utf8') + else: + b64_bytes = content.encode('ascii') + bcontent = base64.decodestring(b64_bytes) + except Exception as e: + raise web.HTTPError(400, u'Encoding error saving %s: %s' % (os_path, e)) + with io.open(os_path, 'wb') as f: + f.write(bcontent) + + def _save_directory(self, os_path, model, name='', path=''): + """create a directory""" + if is_hidden(os_path, self.root_dir): + raise web.HTTPError(400, u'Cannot create hidden directory %r' % os_path) + if not os.path.exists(os_path): + os.mkdir(os_path) + elif not os.path.isdir(os_path): + raise web.HTTPError(400, u'Not a directory: %s' % (os_path)) + else: + self.log.debug("Directory %r already exists", os_path) + + def save(self, model, name='', path=''): + """Save the file model and return the model with no content.""" + path = path.strip('/') + + if 'type' not in model: + raise web.HTTPError(400, u'No file type provided') + if 'content' not in model and model['type'] != 'directory': + raise web.HTTPError(400, u'No file content provided') + + # One checkpoint should always exist + if self.file_exists(name, path) and not self.list_checkpoints(name, path): + self.create_checkpoint(name, path) + + new_path = model.get('path', path).strip('/') + new_name = model.get('name', name) + + if path != new_path or name != new_name: + self.rename(name, path, new_name, new_path) + + os_path = self._get_os_path(new_name, new_path) + self.log.debug("Saving %s", os_path) + try: + if model['type'] == 'notebook': + self._save_notebook(os_path, model, new_name, new_path) + elif model['type'] == 'file': + self._save_file(os_path, model, new_name, new_path) + elif model['type'] == 'directory': + self._save_directory(os_path, model, new_name, new_path) + else: + raise web.HTTPError(400, "Unhandled contents type: %s" % model['type']) + except web.HTTPError: + raise + except Exception as e: + raise web.HTTPError(400, u'Unexpected error while saving file: %s %s' % (os_path, e)) + + model = self.get_model(new_name, new_path, content=False) + return model + + def update(self, model, name, path=''): + """Update the file's path and/or name + + For use in PATCH requests, to enable renaming a file without + re-uploading its contents. Only used for renaming at the moment. + """ + path = path.strip('/') + new_name = model.get('name', name) + new_path = model.get('path', path).strip('/') + if path != new_path or name != new_name: + self.rename(name, path, new_name, new_path) + model = self.get_model(new_name, new_path, content=False) + return model + + def delete(self, name, path=''): + """Delete file by name and path.""" + path = path.strip('/') + os_path = self._get_os_path(name, path) + rm = os.unlink + if os.path.isdir(os_path): + listing = os.listdir(os_path) + # don't delete non-empty directories (checkpoints dir doesn't count) + if listing and listing != [self.checkpoint_dir]: + raise web.HTTPError(400, u'Directory %s not empty' % os_path) + elif not os.path.isfile(os_path): + raise web.HTTPError(404, u'File does not exist: %s' % os_path) + + # clear checkpoints + for checkpoint in self.list_checkpoints(name, path): + checkpoint_id = checkpoint['id'] + cp_path = self.get_checkpoint_path(checkpoint_id, name, path) + if os.path.isfile(cp_path): + self.log.debug("Unlinking checkpoint %s", cp_path) + os.unlink(cp_path) + + if os.path.isdir(os_path): + self.log.debug("Removing directory %s", os_path) + shutil.rmtree(os_path) + else: + self.log.debug("Unlinking file %s", os_path) + rm(os_path) + + def rename(self, old_name, old_path, new_name, new_path): + """Rename a file.""" + old_path = old_path.strip('/') + new_path = new_path.strip('/') + if new_name == old_name and new_path == old_path: + return + + new_os_path = self._get_os_path(new_name, new_path) + old_os_path = self._get_os_path(old_name, old_path) + + # Should we proceed with the move? + if os.path.isfile(new_os_path): + raise web.HTTPError(409, u'File with name already exists: %s' % new_os_path) + + # Move the file + try: + shutil.move(old_os_path, new_os_path) + except Exception as e: + raise web.HTTPError(500, u'Unknown error renaming file: %s %s' % (old_os_path, e)) + + # Move the checkpoints + old_checkpoints = self.list_checkpoints(old_name, old_path) + for cp in old_checkpoints: + checkpoint_id = cp['id'] + old_cp_path = self.get_checkpoint_path(checkpoint_id, old_name, old_path) + new_cp_path = self.get_checkpoint_path(checkpoint_id, new_name, new_path) + if os.path.isfile(old_cp_path): + self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path) + shutil.move(old_cp_path, new_cp_path) + + # Checkpoint-related utilities + + def get_checkpoint_path(self, checkpoint_id, name, path=''): + """find the path to a checkpoint""" + path = path.strip('/') + basename, ext = os.path.splitext(name) + filename = u"{name}-{checkpoint_id}{ext}".format( + name=basename, + checkpoint_id=checkpoint_id, + ext=ext, + ) + os_path = self._get_os_path(path=path) + cp_dir = os.path.join(os_path, self.checkpoint_dir) + ensure_dir_exists(cp_dir) + cp_path = os.path.join(cp_dir, filename) + return cp_path + + def get_checkpoint_model(self, checkpoint_id, name, path=''): + """construct the info dict for a given checkpoint""" + path = path.strip('/') + cp_path = self.get_checkpoint_path(checkpoint_id, name, path) + stats = os.stat(cp_path) + last_modified = tz.utcfromtimestamp(stats.st_mtime) + info = dict( + id = checkpoint_id, + last_modified = last_modified, + ) + return info + + # public checkpoint API + + def create_checkpoint(self, name, path=''): + """Create a checkpoint from the current state of a file""" + path = path.strip('/') + src_path = self._get_os_path(name, path) + # only the one checkpoint ID: + checkpoint_id = u"checkpoint" + cp_path = self.get_checkpoint_path(checkpoint_id, name, path) + self.log.debug("creating checkpoint for %s", name) + self._copy(src_path, cp_path) + + # return the checkpoint info + return self.get_checkpoint_model(checkpoint_id, name, path) + + def list_checkpoints(self, name, path=''): + """list the checkpoints for a given file + + This contents manager currently only supports one checkpoint per file. + """ + path = path.strip('/') + checkpoint_id = "checkpoint" + os_path = self.get_checkpoint_path(checkpoint_id, name, path) + if not os.path.exists(os_path): + return [] + else: + return [self.get_checkpoint_model(checkpoint_id, name, path)] + + + def restore_checkpoint(self, checkpoint_id, name, path=''): + """restore a file to a checkpointed state""" + path = path.strip('/') + self.log.info("restoring %s from checkpoint %s", name, checkpoint_id) + nb_path = self._get_os_path(name, path) + cp_path = self.get_checkpoint_path(checkpoint_id, name, path) + if not os.path.isfile(cp_path): + self.log.debug("checkpoint file does not exist: %s", cp_path) + raise web.HTTPError(404, + u'checkpoint does not exist: %s-%s' % (name, checkpoint_id) + ) + # ensure notebook is readable (never restore from an unreadable notebook) + if cp_path.endswith('.ipynb'): + with io.open(cp_path, 'r', encoding='utf-8') as f: + current.read(f, u'json') + self._copy(cp_path, nb_path) + self.log.debug("copying %s -> %s", cp_path, nb_path) + + def delete_checkpoint(self, checkpoint_id, name, path=''): + """delete a file's checkpoint""" + path = path.strip('/') + cp_path = self.get_checkpoint_path(checkpoint_id, name, path) + if not os.path.isfile(cp_path): + raise web.HTTPError(404, + u'Checkpoint does not exist: %s%s-%s' % (path, name, checkpoint_id) + ) + self.log.debug("unlinking %s", cp_path) + os.unlink(cp_path) + + def info_string(self): + return "Serving notebooks from local directory: %s" % self.root_dir + + def get_kernel_path(self, name, path='', model=None): + """Return the initial working dir a kernel associated with a given notebook""" + return os.path.join(self.root_dir, path) diff --git a/IPython/html/services/contents/handlers.py b/IPython/html/services/contents/handlers.py new file mode 100644 index 0000000..72860ad --- /dev/null +++ b/IPython/html/services/contents/handlers.py @@ -0,0 +1,286 @@ +"""Tornado handlers for the contents web service.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +import json + +from tornado import web + +from IPython.html.utils import url_path_join, url_escape +from IPython.utils.jsonutil import date_default + +from IPython.html.base.handlers import (IPythonHandler, json_errors, + file_path_regex, path_regex, + file_name_regex) + + +def sort_key(model): + """key function for case-insensitive sort by name and type""" + iname = model['name'].lower() + type_key = { + 'directory' : '0', + 'notebook' : '1', + 'file' : '2', + }.get(model['type'], '9') + return u'%s%s' % (type_key, iname) + +class ContentsHandler(IPythonHandler): + + SUPPORTED_METHODS = (u'GET', u'PUT', u'PATCH', u'POST', u'DELETE') + + def location_url(self, name, path): + """Return the full URL location of a file. + + Parameters + ---------- + name : unicode + The base name of the file, such as "foo.ipynb". + path : unicode + The API path of the file, such as "foo/bar". + """ + return url_escape(url_path_join( + self.base_url, 'api', 'contents', path, name + )) + + def _finish_model(self, model, location=True): + """Finish a JSON request with a model, setting relevant headers, etc.""" + if location: + location = self.location_url(model['name'], model['path']) + self.set_header('Location', location) + self.set_header('Last-Modified', model['last_modified']) + self.finish(json.dumps(model, default=date_default)) + + @web.authenticated + @json_errors + def get(self, path='', name=None): + """Return a model for a file or directory. + + A directory model contains a list of models (without content) + of the files and directories it contains. + """ + path = path or '' + model = self.contents_manager.get_model(name=name, path=path) + if model['type'] == 'directory': + # group listing by type, then by name (case-insensitive) + # FIXME: sorting should be done in the frontends + model['content'].sort(key=sort_key) + self._finish_model(model, location=False) + + @web.authenticated + @json_errors + def patch(self, path='', name=None): + """PATCH renames a notebook without re-uploading content.""" + cm = self.contents_manager + if name is None: + raise web.HTTPError(400, u'Filename missing') + model = self.get_json_body() + if model is None: + raise web.HTTPError(400, u'JSON body missing') + model = cm.update(model, name, path) + self._finish_model(model) + + def _copy(self, copy_from, path, copy_to=None): + """Copy a file, optionally specifying the new name. + """ + self.log.info(u"Copying {copy_from} to {path}/{copy_to}".format( + copy_from=copy_from, + path=path, + copy_to=copy_to or '', + )) + model = self.contents_manager.copy(copy_from, copy_to, path) + self.set_status(201) + self._finish_model(model) + + def _upload(self, model, path, name=None): + """Handle upload of a new file + + If name specified, create it in path/name, + otherwise create a new untitled file in path. + """ + self.log.info(u"Uploading file to %s/%s", path, name or '') + if name: + model['name'] = name + + model = self.contents_manager.create_file(model, path) + self.set_status(201) + self._finish_model(model) + + def _create_empty_file(self, path, name=None, ext='.ipynb'): + """Create an empty file in path + + If name specified, create it in path/name. + """ + self.log.info(u"Creating new file in %s/%s", path, name or '') + model = {} + if name: + model['name'] = name + model = self.contents_manager.create_file(model, path=path, ext=ext) + self.set_status(201) + self._finish_model(model) + + def _save(self, model, path, name): + """Save an existing file.""" + self.log.info(u"Saving file at %s/%s", path, name) + model = self.contents_manager.save(model, name, path) + if model['path'] != path.strip('/') or model['name'] != name: + # a rename happened, set Location header + location = True + else: + location = False + self._finish_model(model, location) + + @web.authenticated + @json_errors + def post(self, path='', name=None): + """Create a new file or directory in the specified path. + + POST creates new files or directories. The server always decides on the name. + + POST /api/contents/path + New untitled notebook in path. If content specified, upload a + notebook, otherwise start empty. + POST /api/contents/path + with body {"copy_from" : "OtherNotebook.ipynb"} + New copy of OtherNotebook in path + """ + + if name is not None: + path = u'{}/{}'.format(path, name) + + cm = self.contents_manager + + if cm.file_exists(path): + raise web.HTTPError(400, "Cannot POST to existing files, use PUT instead.") + + if not cm.path_exists(path): + raise web.HTTPError(404, "No such directory: %s" % path) + + model = self.get_json_body() + + if model is not None: + copy_from = model.get('copy_from') + ext = model.get('ext', '.ipynb') + if model.get('content') is not None: + if copy_from: + raise web.HTTPError(400, "Can't upload and copy at the same time.") + self._upload(model, path) + elif copy_from: + self._copy(copy_from, path) + else: + self._create_empty_file(path, ext=ext) + else: + self._create_empty_file(path) + + @web.authenticated + @json_errors + def put(self, path='', name=None): + """Saves the file in the location specified by name and path. + + PUT is very similar to POST, but the requester specifies the name, + whereas with POST, the server picks the name. + + PUT /api/contents/path/Name.ipynb + Save notebook at ``path/Name.ipynb``. Notebook structure is specified + in `content` key of JSON request body. If content is not specified, + create a new empty notebook. + PUT /api/contents/path/Name.ipynb + with JSON body:: + + { + "copy_from" : "[path/to/]OtherNotebook.ipynb" + } + + Copy OtherNotebook to Name + """ + if name is None: + raise web.HTTPError(400, "name must be specified with PUT.") + + model = self.get_json_body() + if model: + copy_from = model.get('copy_from') + if copy_from: + if model.get('content'): + raise web.HTTPError(400, "Can't upload and copy at the same time.") + self._copy(copy_from, path, name) + elif self.contents_manager.file_exists(name, path): + self._save(model, path, name) + else: + self._upload(model, path, name) + else: + self._create_empty_file(path, name) + + @web.authenticated + @json_errors + def delete(self, path='', name=None): + """delete a file in the given path""" + cm = self.contents_manager + self.log.warn('delete %s:%s', path, name) + cm.delete(name, path) + self.set_status(204) + self.finish() + + +class CheckpointsHandler(IPythonHandler): + + SUPPORTED_METHODS = ('GET', 'POST') + + @web.authenticated + @json_errors + def get(self, path='', name=None): + """get lists checkpoints for a file""" + cm = self.contents_manager + checkpoints = cm.list_checkpoints(name, path) + data = json.dumps(checkpoints, default=date_default) + self.finish(data) + + @web.authenticated + @json_errors + def post(self, path='', name=None): + """post creates a new checkpoint""" + cm = self.contents_manager + checkpoint = cm.create_checkpoint(name, path) + data = json.dumps(checkpoint, default=date_default) + location = url_path_join(self.base_url, 'api/contents', + path, name, 'checkpoints', checkpoint['id']) + self.set_header('Location', url_escape(location)) + self.set_status(201) + self.finish(data) + + +class ModifyCheckpointsHandler(IPythonHandler): + + SUPPORTED_METHODS = ('POST', 'DELETE') + + @web.authenticated + @json_errors + def post(self, path, name, checkpoint_id): + """post restores a file from a checkpoint""" + cm = self.contents_manager + cm.restore_checkpoint(checkpoint_id, name, path) + self.set_status(204) + self.finish() + + @web.authenticated + @json_errors + def delete(self, path, name, checkpoint_id): + """delete clears a checkpoint for a given file""" + cm = self.contents_manager + cm.delete_checkpoint(checkpoint_id, name, path) + self.set_status(204) + self.finish() + +#----------------------------------------------------------------------------- +# URL to handler mappings +#----------------------------------------------------------------------------- + + +_checkpoint_id_regex = r"(?P[\w-]+)" + +default_handlers = [ + (r"/api/contents%s/checkpoints" % file_path_regex, CheckpointsHandler), + (r"/api/contents%s/checkpoints/%s" % (file_path_regex, _checkpoint_id_regex), + ModifyCheckpointsHandler), + (r"/api/contents%s" % file_path_regex, ContentsHandler), + (r"/api/contents%s" % path_regex, ContentsHandler), +] diff --git a/IPython/html/services/contents/manager.py b/IPython/html/services/contents/manager.py new file mode 100644 index 0000000..e6a11ed --- /dev/null +++ b/IPython/html/services/contents/manager.py @@ -0,0 +1,333 @@ +"""A base class for contents managers.""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +from fnmatch import fnmatch +import itertools +import os + +from tornado.web import HTTPError + +from IPython.config.configurable import LoggingConfigurable +from IPython.nbformat import current, sign +from IPython.utils.traitlets import Instance, Unicode, List + + +class ContentsManager(LoggingConfigurable): + """Base class for serving files and directories. + + This serves any text or binary file, + as well as directories, + with special handling for JSON notebook documents. + + Most APIs take a path argument, + which is always an API-style unicode path, + and always refers to a directory. + + - unicode, not url-escaped + - '/'-separated + - leading and trailing '/' will be stripped + - if unspecified, path defaults to '', + indicating the root path. + + name is also unicode, and refers to a specfic target: + + - unicode, not url-escaped + - must not contain '/' + - It refers to an individual filename + - It may refer to a directory name, + in the case of listing or creating directories. + + """ + + notary = Instance(sign.NotebookNotary) + def _notary_default(self): + return sign.NotebookNotary(parent=self) + + hide_globs = List(Unicode, [ + u'__pycache__', '*.pyc', '*.pyo', + '.DS_Store', '*.so', '*.dylib', '*~', + ], config=True, help=""" + Glob patterns to hide in file and directory listings. + """) + + untitled_notebook = Unicode("Untitled", config=True, + help="The base name used when creating untitled notebooks." + ) + + untitled_file = Unicode("untitled", config=True, + help="The base name used when creating untitled files." + ) + + untitled_directory = Unicode("Untitled Folder", config=True, + help="The base name used when creating untitled directories." + ) + + # ContentsManager API part 1: methods that must be + # implemented in subclasses. + + def path_exists(self, path): + """Does the API-style path (directory) actually exist? + + Like os.path.isdir + + Override this method in subclasses. + + Parameters + ---------- + path : string + The path to check + + Returns + ------- + exists : bool + Whether the path does indeed exist. + """ + raise NotImplementedError + + def is_hidden(self, path): + """Does the API style path correspond to a hidden directory or file? + + Parameters + ---------- + path : string + The path to check. This is an API path (`/` separated, + relative to root dir). + + Returns + ------- + hidden : bool + Whether the path is hidden. + + """ + raise NotImplementedError + + def file_exists(self, name, path=''): + """Does a file exist at the given name and path? + + Like os.path.isfile + + Override this method in subclasses. + + Parameters + ---------- + name : string + The name of the file you are checking. + path : string + The relative path to the file's directory (with '/' as separator) + + Returns + ------- + exists : bool + Whether the file exists. + """ + raise NotImplementedError('must be implemented in a subclass') + + def exists(self, name, path=''): + """Does a file or directory exist at the given name and path? + + Like os.path.exists + + Parameters + ---------- + name : string + The name of the file you are checking. + path : string + The relative path to the file's directory (with '/' as separator) + + Returns + ------- + exists : bool + Whether the target exists. + """ + return self.file_exists(name, path) or self.path_exists("%s/%s" % (path, name)) + + def get_model(self, name, path='', content=True): + """Get the model of a file or directory with or without content.""" + raise NotImplementedError('must be implemented in a subclass') + + def save(self, model, name, path=''): + """Save the file or directory and return the model with no content.""" + raise NotImplementedError('must be implemented in a subclass') + + def update(self, model, name, path=''): + """Update the file or directory and return the model with no content. + + For use in PATCH requests, to enable renaming a file without + re-uploading its contents. Only used for renaming at the moment. + """ + raise NotImplementedError('must be implemented in a subclass') + + def delete(self, name, path=''): + """Delete file or directory by name and path.""" + raise NotImplementedError('must be implemented in a subclass') + + def create_checkpoint(self, name, path=''): + """Create a checkpoint of the current state of a file + + Returns a checkpoint_id for the new checkpoint. + """ + raise NotImplementedError("must be implemented in a subclass") + + def list_checkpoints(self, name, path=''): + """Return a list of checkpoints for a given file""" + return [] + + def restore_checkpoint(self, checkpoint_id, name, path=''): + """Restore a file from one of its checkpoints""" + raise NotImplementedError("must be implemented in a subclass") + + def delete_checkpoint(self, checkpoint_id, name, path=''): + """delete a checkpoint for a file""" + raise NotImplementedError("must be implemented in a subclass") + + # ContentsManager API part 2: methods that have useable default + # implementations, but can be overridden in subclasses. + + def info_string(self): + return "Serving contents" + + def get_kernel_path(self, name, path='', model=None): + """ Return the path to start kernel in """ + return path + + def increment_filename(self, filename, path=''): + """Increment a filename until it is unique. + + Parameters + ---------- + filename : unicode + The name of a file, including extension + path : unicode + The API path of the target's directory + + Returns + ------- + name : unicode + A filename that is unique, based on the input filename. + """ + path = path.strip('/') + basename, ext = os.path.splitext(filename) + for i in itertools.count(): + name = u'{basename}{i}{ext}'.format(basename=basename, i=i, + ext=ext) + if not self.file_exists(name, path): + break + return name + + def create_file(self, model=None, path='', ext='.ipynb'): + """Create a new file or directory and return its model with no content.""" + path = path.strip('/') + if model is None: + model = {} + if 'content' not in model and model.get('type', None) != 'directory': + if ext == '.ipynb': + metadata = current.new_metadata(name=u'') + model['content'] = current.new_notebook(metadata=metadata) + model['type'] = 'notebook' + model['format'] = 'json' + else: + model['content'] = '' + model['type'] = 'file' + model['format'] = 'text' + if 'name' not in model: + if model['type'] == 'directory': + untitled = self.untitled_directory + elif model['type'] == 'notebook': + untitled = self.untitled_notebook + elif model['type'] == 'file': + untitled = self.untitled_file + else: + raise HTTPError(400, "Unexpected model type: %r" % model['type']) + model['name'] = self.increment_filename(untitled + ext, path) + + model['path'] = path + model = self.save(model, model['name'], model['path']) + return model + + def copy(self, from_name, to_name=None, path=''): + """Copy an existing file and return its new model. + + If to_name not specified, increment `from_name-Copy#.ext`. + + copy_from can be a full path to a file, + or just a base name. If a base name, `path` is used. + """ + path = path.strip('/') + if '/' in from_name: + from_path, from_name = from_name.rsplit('/', 1) + else: + from_path = path + model = self.get_model(from_name, from_path) + if model['type'] == 'directory': + raise HTTPError(400, "Can't copy directories") + if not to_name: + base, ext = os.path.splitext(from_name) + copy_name = u'{0}-Copy{1}'.format(base, ext) + to_name = self.increment_filename(copy_name, path) + model['name'] = to_name + model['path'] = path + model = self.save(model, to_name, path) + return model + + def log_info(self): + self.log.info(self.info_string()) + + def trust_notebook(self, name, path=''): + """Explicitly trust a notebook + + Parameters + ---------- + name : string + The filename of the notebook + path : string + The notebook's directory + """ + model = self.get_model(name, path) + nb = model['content'] + self.log.warn("Trusting notebook %s/%s", path, name) + self.notary.mark_cells(nb, True) + self.save(model, name, path) + + def check_and_sign(self, nb, name='', path=''): + """Check for trusted cells, and sign the notebook. + + Called as a part of saving notebooks. + + Parameters + ---------- + nb : dict + The notebook object (in nbformat.current format) + name : string + The filename of the notebook (for logging) + path : string + The notebook's directory (for logging) + """ + if self.notary.check_cells(nb): + self.notary.sign(nb) + else: + self.log.warn("Saving untrusted notebook %s/%s", path, name) + + def mark_trusted_cells(self, nb, name='', path=''): + """Mark cells as trusted if the notebook signature matches. + + Called as a part of loading notebooks. + + Parameters + ---------- + nb : dict + The notebook object (in nbformat.current format) + name : string + The filename of the notebook (for logging) + path : string + The notebook's directory (for logging) + """ + trusted = self.notary.check_signature(nb) + if not trusted: + self.log.warn("Notebook %s/%s is not trusted", path, name) + self.notary.mark_cells(nb, trusted) + + def should_list(self, name): + """Should this file/directory name be displayed in a listing?""" + return not any(fnmatch(name, glob) for glob in self.hide_globs) diff --git a/IPython/html/services/notebooks/tests/__init__.py b/IPython/html/services/contents/tests/__init__.py similarity index 100% rename from IPython/html/services/notebooks/tests/__init__.py rename to IPython/html/services/contents/tests/__init__.py diff --git a/IPython/html/services/notebooks/tests/test_notebooks_api.py b/IPython/html/services/contents/tests/test_contents_api.py similarity index 50% rename from IPython/html/services/notebooks/tests/test_notebooks_api.py rename to IPython/html/services/contents/tests/test_contents_api.py index c8c82e8..bac91de 100644 --- a/IPython/html/services/notebooks/tests/test_notebooks_api.py +++ b/IPython/html/services/contents/tests/test_contents_api.py @@ -1,6 +1,7 @@ # coding: utf-8 -"""Test the notebooks webservice API.""" +"""Test the contents webservice API.""" +import base64 import io import json import os @@ -21,23 +22,21 @@ from IPython.utils import py3compat from IPython.utils.data import uniq_stable -# TODO: Remove this after we create the contents web service and directories are -# no longer listed by the notebook web service. -def notebooks_only(nb_list): - return [nb for nb in nb_list if nb['type']=='notebook'] +def notebooks_only(dir_model): + return [nb for nb in dir_model['content'] if nb['type']=='notebook'] -def dirs_only(nb_list): - return [x for x in nb_list if x['type']=='directory'] +def dirs_only(dir_model): + return [x for x in dir_model['content'] if x['type']=='directory'] -class NBAPI(object): - """Wrapper for notebook API calls.""" +class API(object): + """Wrapper for contents API calls.""" def __init__(self, base_url): self.base_url = base_url def _req(self, verb, path, body=None): response = requests.request(verb, - url_path_join(self.base_url, 'api/notebooks', path), + url_path_join(self.base_url, 'api/contents', path), data=body, ) response.raise_for_status() @@ -49,8 +48,11 @@ class NBAPI(object): def read(self, name, path='/'): return self._req('GET', url_path_join(path, name)) - def create_untitled(self, path='/'): - return self._req('POST', path) + def create_untitled(self, path='/', ext=None): + body = None + if ext: + body = json.dumps({'ext': ext}) + return self._req('POST', path, body) def upload_untitled(self, body, path='/'): return self._req('POST', path, body) @@ -65,6 +67,9 @@ class NBAPI(object): def upload(self, name, body, path='/'): return self._req('PUT', url_path_join(path, name), body) + def mkdir(self, name, path='/'): + return self._req('PUT', url_path_join(path, name), json.dumps({'type': 'directory'})) + def copy(self, copy_from, copy_to, path='/'): body = json.dumps({'copy_from':copy_from}) return self._req('PUT', url_path_join(path, copy_to), body) @@ -112,8 +117,20 @@ class APITest(NotebookTestBase): del dirs[0] # remove '' top_level_dirs = {normalize('NFC', d.split('/')[0]) for d in dirs} + @staticmethod + def _blob_for_name(name): + return name.encode('utf-8') + b'\xFF' + + @staticmethod + def _txt_for_name(name): + return u'%s text file' % name + def setUp(self): nbdir = self.notebook_dir.name + self.blob = os.urandom(100) + self.b64_blob = base64.encodestring(self.blob).decode('ascii') + + for d in (self.dirs + self.hidden_dirs): d.replace('/', os.sep) @@ -122,12 +139,22 @@ class APITest(NotebookTestBase): for d, name in self.dirs_nbs: d = d.replace('/', os.sep) + # create a notebook with io.open(pjoin(nbdir, d, '%s.ipynb' % name), 'w', encoding='utf-8') as f: nb = new_notebook(name=name) write(nb, f, format='ipynb') - self.nb_api = NBAPI(self.base_url()) + # create a text file + with io.open(pjoin(nbdir, d, '%s.txt' % name), 'w', + encoding='utf-8') as f: + f.write(self._txt_for_name(name)) + + # create a binary file + with io.open(pjoin(nbdir, d, '%s.blob' % name), 'wb') as f: + f.write(self._blob_for_name(name)) + + self.api = API(self.base_url()) def tearDown(self): nbdir = self.notebook_dir.name @@ -139,175 +166,287 @@ class APITest(NotebookTestBase): os.unlink(pjoin(nbdir, 'inroot.ipynb')) def test_list_notebooks(self): - nbs = notebooks_only(self.nb_api.list().json()) + nbs = notebooks_only(self.api.list().json()) self.assertEqual(len(nbs), 1) self.assertEqual(nbs[0]['name'], 'inroot.ipynb') - nbs = notebooks_only(self.nb_api.list('/Directory with spaces in/').json()) + nbs = notebooks_only(self.api.list('/Directory with spaces in/').json()) self.assertEqual(len(nbs), 1) self.assertEqual(nbs[0]['name'], 'inspace.ipynb') - nbs = notebooks_only(self.nb_api.list(u'/unicodé/').json()) + nbs = notebooks_only(self.api.list(u'/unicodé/').json()) self.assertEqual(len(nbs), 1) self.assertEqual(nbs[0]['name'], 'innonascii.ipynb') self.assertEqual(nbs[0]['path'], u'unicodé') - nbs = notebooks_only(self.nb_api.list('/foo/bar/').json()) + nbs = notebooks_only(self.api.list('/foo/bar/').json()) self.assertEqual(len(nbs), 1) self.assertEqual(nbs[0]['name'], 'baz.ipynb') self.assertEqual(nbs[0]['path'], 'foo/bar') - nbs = notebooks_only(self.nb_api.list('foo').json()) + nbs = notebooks_only(self.api.list('foo').json()) self.assertEqual(len(nbs), 4) nbnames = { normalize('NFC', n['name']) for n in nbs } expected = [ u'a.ipynb', u'b.ipynb', u'name with spaces.ipynb', u'unicodé.ipynb'] expected = { normalize('NFC', name) for name in expected } self.assertEqual(nbnames, expected) - - nbs = notebooks_only(self.nb_api.list('ordering').json()) + + nbs = notebooks_only(self.api.list('ordering').json()) nbnames = [n['name'] for n in nbs] expected = ['A.ipynb', 'b.ipynb', 'C.ipynb'] self.assertEqual(nbnames, expected) def test_list_dirs(self): - dirs = dirs_only(self.nb_api.list().json()) + dirs = dirs_only(self.api.list().json()) dir_names = {normalize('NFC', d['name']) for d in dirs} self.assertEqual(dir_names, self.top_level_dirs) # Excluding hidden dirs def test_list_nonexistant_dir(self): with assert_http_error(404): - self.nb_api.list('nonexistant') + self.api.list('nonexistant') - def test_get_contents(self): + def test_get_nb_contents(self): for d, name in self.dirs_nbs: - nb = self.nb_api.read('%s.ipynb' % name, d+'/').json() + nb = self.api.read('%s.ipynb' % name, d+'/').json() self.assertEqual(nb['name'], u'%s.ipynb' % name) + self.assertEqual(nb['type'], 'notebook') + self.assertIn('content', nb) + self.assertEqual(nb['format'], 'json') self.assertIn('content', nb) self.assertIn('metadata', nb['content']) self.assertIsInstance(nb['content']['metadata'], dict) + def test_get_contents_no_such_file(self): + # Name that doesn't exist - should be a 404 + with assert_http_error(404): + self.api.read('q.ipynb', 'foo') + + def test_get_text_file_contents(self): + for d, name in self.dirs_nbs: + model = self.api.read(u'%s.txt' % name, d+'/').json() + self.assertEqual(model['name'], u'%s.txt' % name) + self.assertIn('content', model) + self.assertEqual(model['format'], 'text') + self.assertEqual(model['type'], 'file') + self.assertEqual(model['content'], self._txt_for_name(name)) + + # Name that doesn't exist - should be a 404 + with assert_http_error(404): + self.api.read('q.txt', 'foo') + + def test_get_binary_file_contents(self): + for d, name in self.dirs_nbs: + model = self.api.read(u'%s.blob' % name, d+'/').json() + self.assertEqual(model['name'], u'%s.blob' % name) + self.assertIn('content', model) + self.assertEqual(model['format'], 'base64') + self.assertEqual(model['type'], 'file') + b64_data = base64.encodestring(self._blob_for_name(name)).decode('ascii') + self.assertEqual(model['content'], b64_data) + # Name that doesn't exist - should be a 404 with assert_http_error(404): - self.nb_api.read('q.ipynb', 'foo') + self.api.read('q.txt', 'foo') - def _check_nb_created(self, resp, name, path): + def _check_created(self, resp, name, path, type='notebook'): self.assertEqual(resp.status_code, 201) location_header = py3compat.str_to_unicode(resp.headers['Location']) - self.assertEqual(location_header, url_escape(url_path_join(u'/api/notebooks', path, name))) - self.assertEqual(resp.json()['name'], name) - assert os.path.isfile(pjoin( + self.assertEqual(location_header, url_escape(url_path_join(u'/api/contents', path, name))) + rjson = resp.json() + self.assertEqual(rjson['name'], name) + self.assertEqual(rjson['path'], path) + self.assertEqual(rjson['type'], type) + isright = os.path.isdir if type == 'directory' else os.path.isfile + assert isright(pjoin( self.notebook_dir.name, path.replace('/', os.sep), name, )) def test_create_untitled(self): - resp = self.nb_api.create_untitled(path=u'å b') - self._check_nb_created(resp, 'Untitled0.ipynb', u'å b') + resp = self.api.create_untitled(path=u'å b') + self._check_created(resp, 'Untitled0.ipynb', u'å b') # Second time - resp = self.nb_api.create_untitled(path=u'å b') - self._check_nb_created(resp, 'Untitled1.ipynb', u'å b') + resp = self.api.create_untitled(path=u'å b') + self._check_created(resp, 'Untitled1.ipynb', u'å b') # And two directories down - resp = self.nb_api.create_untitled(path='foo/bar') - self._check_nb_created(resp, 'Untitled0.ipynb', 'foo/bar') + resp = self.api.create_untitled(path='foo/bar') + self._check_created(resp, 'Untitled0.ipynb', 'foo/bar') + + def test_create_untitled_txt(self): + resp = self.api.create_untitled(path='foo/bar', ext='.txt') + self._check_created(resp, 'untitled0.txt', 'foo/bar', type='file') + + resp = self.api.read(path='foo/bar', name='untitled0.txt') + model = resp.json() + self.assertEqual(model['type'], 'file') + self.assertEqual(model['format'], 'text') + self.assertEqual(model['content'], '') def test_upload_untitled(self): nb = new_notebook(name='Upload test') - nbmodel = {'content': nb} - resp = self.nb_api.upload_untitled(path=u'å b', + nbmodel = {'content': nb, 'type': 'notebook'} + resp = self.api.upload_untitled(path=u'å b', body=json.dumps(nbmodel)) - self._check_nb_created(resp, 'Untitled0.ipynb', u'å b') + self._check_created(resp, 'Untitled0.ipynb', u'å b') def test_upload(self): nb = new_notebook(name=u'ignored') - nbmodel = {'content': nb} - resp = self.nb_api.upload(u'Upload tést.ipynb', path=u'å b', + nbmodel = {'content': nb, 'type': 'notebook'} + resp = self.api.upload(u'Upload tést.ipynb', path=u'å b', body=json.dumps(nbmodel)) - self._check_nb_created(resp, u'Upload tést.ipynb', u'å b') + self._check_created(resp, u'Upload tést.ipynb', u'å b') + + def test_mkdir(self): + resp = self.api.mkdir(u'New ∂ir', path=u'å b') + self._check_created(resp, u'New ∂ir', u'å b', type='directory') + + def test_mkdir_hidden_400(self): + with assert_http_error(400): + resp = self.api.mkdir(u'.hidden', path=u'å b') + + def test_upload_txt(self): + body = u'ünicode téxt' + model = { + 'content' : body, + 'format' : 'text', + 'type' : 'file', + } + resp = self.api.upload(u'Upload tést.txt', path=u'å b', + body=json.dumps(model)) + + # check roundtrip + resp = self.api.read(path=u'å b', name=u'Upload tést.txt') + model = resp.json() + self.assertEqual(model['type'], 'file') + self.assertEqual(model['format'], 'text') + self.assertEqual(model['content'], body) + + def test_upload_b64(self): + body = b'\xFFblob' + b64body = base64.encodestring(body).decode('ascii') + model = { + 'content' : b64body, + 'format' : 'base64', + 'type' : 'file', + } + resp = self.api.upload(u'Upload tést.blob', path=u'å b', + body=json.dumps(model)) + + # check roundtrip + resp = self.api.read(path=u'å b', name=u'Upload tést.blob') + model = resp.json() + self.assertEqual(model['type'], 'file') + self.assertEqual(model['format'], 'base64') + decoded = base64.decodestring(model['content'].encode('ascii')) + self.assertEqual(decoded, body) def test_upload_v2(self): nb = v2.new_notebook() ws = v2.new_worksheet() nb.worksheets.append(ws) ws.cells.append(v2.new_code_cell(input='print("hi")')) - nbmodel = {'content': nb} - resp = self.nb_api.upload(u'Upload tést.ipynb', path=u'å b', + nbmodel = {'content': nb, 'type': 'notebook'} + resp = self.api.upload(u'Upload tést.ipynb', path=u'å b', body=json.dumps(nbmodel)) - self._check_nb_created(resp, u'Upload tést.ipynb', u'å b') - resp = self.nb_api.read(u'Upload tést.ipynb', u'å b') + self._check_created(resp, u'Upload tést.ipynb', u'å b') + resp = self.api.read(u'Upload tést.ipynb', u'å b') data = resp.json() self.assertEqual(data['content']['nbformat'], current.nbformat) self.assertEqual(data['content']['orig_nbformat'], 2) def test_copy_untitled(self): - resp = self.nb_api.copy_untitled(u'ç d.ipynb', path=u'å b') - self._check_nb_created(resp, u'ç d-Copy0.ipynb', u'å b') + resp = self.api.copy_untitled(u'ç d.ipynb', path=u'å b') + self._check_created(resp, u'ç d-Copy0.ipynb', u'å b') def test_copy(self): - resp = self.nb_api.copy(u'ç d.ipynb', u'cøpy.ipynb', path=u'å b') - self._check_nb_created(resp, u'cøpy.ipynb', u'å b') + resp = self.api.copy(u'ç d.ipynb', u'cøpy.ipynb', path=u'å b') + self._check_created(resp, u'cøpy.ipynb', u'å b') + + def test_copy_path(self): + resp = self.api.copy(u'foo/a.ipynb', u'cøpyfoo.ipynb', path=u'å b') + self._check_created(resp, u'cøpyfoo.ipynb', u'å b') + + def test_copy_dir_400(self): + # can't copy directories + with assert_http_error(400): + resp = self.api.copy(u'å b', u'å c') def test_delete(self): for d, name in self.dirs_nbs: - resp = self.nb_api.delete('%s.ipynb' % name, d) + resp = self.api.delete('%s.ipynb' % name, d) self.assertEqual(resp.status_code, 204) for d in self.dirs + ['/']: - nbs = notebooks_only(self.nb_api.list(d).json()) + nbs = notebooks_only(self.api.list(d).json()) self.assertEqual(len(nbs), 0) + def test_delete_dirs(self): + # depth-first delete everything, so we don't try to delete empty directories + for name in sorted(self.dirs + ['/'], key=len, reverse=True): + listing = self.api.list(name).json()['content'] + for model in listing: + self.api.delete(model['name'], model['path']) + listing = self.api.list('/').json()['content'] + self.assertEqual(listing, []) + + def test_delete_non_empty_dir(self): + """delete non-empty dir raises 400""" + with assert_http_error(400): + self.api.delete(u'å b') + def test_rename(self): - resp = self.nb_api.rename('a.ipynb', 'foo', 'z.ipynb') + resp = self.api.rename('a.ipynb', 'foo', 'z.ipynb') self.assertEqual(resp.headers['Location'].split('/')[-1], 'z.ipynb') self.assertEqual(resp.json()['name'], 'z.ipynb') assert os.path.isfile(pjoin(self.notebook_dir.name, 'foo', 'z.ipynb')) - nbs = notebooks_only(self.nb_api.list('foo').json()) + nbs = notebooks_only(self.api.list('foo').json()) nbnames = set(n['name'] for n in nbs) self.assertIn('z.ipynb', nbnames) self.assertNotIn('a.ipynb', nbnames) def test_rename_existing(self): with assert_http_error(409): - self.nb_api.rename('a.ipynb', 'foo', 'b.ipynb') + self.api.rename('a.ipynb', 'foo', 'b.ipynb') def test_save(self): - resp = self.nb_api.read('a.ipynb', 'foo') + resp = self.api.read('a.ipynb', 'foo') nbcontent = json.loads(resp.text)['content'] nb = to_notebook_json(nbcontent) ws = new_worksheet() nb.worksheets = [ws] ws.cells.append(new_heading_cell(u'Created by test ³')) - nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb} - resp = self.nb_api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) + nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb, 'type': 'notebook'} + resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) nbfile = pjoin(self.notebook_dir.name, 'foo', 'a.ipynb') with io.open(nbfile, 'r', encoding='utf-8') as f: newnb = read(f, format='ipynb') self.assertEqual(newnb.worksheets[0].cells[0].source, u'Created by test ³') - nbcontent = self.nb_api.read('a.ipynb', 'foo').json()['content'] + nbcontent = self.api.read('a.ipynb', 'foo').json()['content'] newnb = to_notebook_json(nbcontent) self.assertEqual(newnb.worksheets[0].cells[0].source, u'Created by test ³') # Save and rename - nbmodel= {'name': 'a2.ipynb', 'path':'foo/bar', 'content': nb} - resp = self.nb_api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) + nbmodel= {'name': 'a2.ipynb', 'path':'foo/bar', 'content': nb, 'type': 'notebook'} + resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) saved = resp.json() self.assertEqual(saved['name'], 'a2.ipynb') self.assertEqual(saved['path'], 'foo/bar') assert os.path.isfile(pjoin(self.notebook_dir.name,'foo','bar','a2.ipynb')) assert not os.path.isfile(pjoin(self.notebook_dir.name, 'foo', 'a.ipynb')) with assert_http_error(404): - self.nb_api.read('a.ipynb', 'foo') + self.api.read('a.ipynb', 'foo') def test_checkpoints(self): - resp = self.nb_api.read('a.ipynb', 'foo') - r = self.nb_api.new_checkpoint('a.ipynb', 'foo') + resp = self.api.read('a.ipynb', 'foo') + r = self.api.new_checkpoint('a.ipynb', 'foo') self.assertEqual(r.status_code, 201) cp1 = r.json() self.assertEqual(set(cp1), {'id', 'last_modified'}) @@ -321,27 +460,26 @@ class APITest(NotebookTestBase): hcell = new_heading_cell('Created by test') ws.cells.append(hcell) # Save - nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb} - resp = self.nb_api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) + nbmodel= {'name': 'a.ipynb', 'path':'foo', 'content': nb, 'type': 'notebook'} + resp = self.api.save('a.ipynb', path='foo', body=json.dumps(nbmodel)) # List checkpoints - cps = self.nb_api.get_checkpoints('a.ipynb', 'foo').json() + cps = self.api.get_checkpoints('a.ipynb', 'foo').json() self.assertEqual(cps, [cp1]) - nbcontent = self.nb_api.read('a.ipynb', 'foo').json()['content'] + nbcontent = self.api.read('a.ipynb', 'foo').json()['content'] nb = to_notebook_json(nbcontent) self.assertEqual(nb.worksheets[0].cells[0].source, 'Created by test') # Restore cp1 - r = self.nb_api.restore_checkpoint('a.ipynb', 'foo', cp1['id']) + r = self.api.restore_checkpoint('a.ipynb', 'foo', cp1['id']) self.assertEqual(r.status_code, 204) - nbcontent = self.nb_api.read('a.ipynb', 'foo').json()['content'] + nbcontent = self.api.read('a.ipynb', 'foo').json()['content'] nb = to_notebook_json(nbcontent) self.assertEqual(nb.worksheets, []) # Delete cp1 - r = self.nb_api.delete_checkpoint('a.ipynb', 'foo', cp1['id']) + r = self.api.delete_checkpoint('a.ipynb', 'foo', cp1['id']) self.assertEqual(r.status_code, 204) - cps = self.nb_api.get_checkpoints('a.ipynb', 'foo').json() + cps = self.api.get_checkpoints('a.ipynb', 'foo').json() self.assertEqual(cps, []) - diff --git a/IPython/html/services/notebooks/tests/test_nbmanager.py b/IPython/html/services/contents/tests/test_manager.py similarity index 55% rename from IPython/html/services/notebooks/tests/test_nbmanager.py rename to IPython/html/services/contents/tests/test_manager.py index bc03a87..e58a895 100644 --- a/IPython/html/services/notebooks/tests/test_nbmanager.py +++ b/IPython/html/services/contents/tests/test_manager.py @@ -15,74 +15,74 @@ from IPython.utils.tempdir import TemporaryDirectory from IPython.utils.traitlets import TraitError from IPython.html.utils import url_path_join -from ..filenbmanager import FileNotebookManager -from ..nbmanager import NotebookManager +from ..filemanager import FileContentsManager +from ..manager import ContentsManager -class TestFileNotebookManager(TestCase): +class TestFileContentsManager(TestCase): - def test_nb_dir(self): + def test_root_dir(self): with TemporaryDirectory() as td: - fm = FileNotebookManager(notebook_dir=td) - self.assertEqual(fm.notebook_dir, td) + fm = FileContentsManager(root_dir=td) + self.assertEqual(fm.root_dir, td) - def test_missing_nb_dir(self): + def test_missing_root_dir(self): with TemporaryDirectory() as td: - nbdir = os.path.join(td, 'notebook', 'dir', 'is', 'missing') - self.assertRaises(TraitError, FileNotebookManager, notebook_dir=nbdir) + root = os.path.join(td, 'notebook', 'dir', 'is', 'missing') + self.assertRaises(TraitError, FileContentsManager, root_dir=root) - def test_invalid_nb_dir(self): + def test_invalid_root_dir(self): with NamedTemporaryFile() as tf: - self.assertRaises(TraitError, FileNotebookManager, notebook_dir=tf.name) + self.assertRaises(TraitError, FileContentsManager, root_dir=tf.name) def test_get_os_path(self): # full filesystem path should be returned with correct operating system # separators. with TemporaryDirectory() as td: - nbdir = td - fm = FileNotebookManager(notebook_dir=nbdir) + root = td + fm = FileContentsManager(root_dir=root) path = fm._get_os_path('test.ipynb', '/path/to/notebook/') rel_path_list = '/path/to/notebook/test.ipynb'.split('/') - fs_path = os.path.join(fm.notebook_dir, *rel_path_list) + fs_path = os.path.join(fm.root_dir, *rel_path_list) self.assertEqual(path, fs_path) - fm = FileNotebookManager(notebook_dir=nbdir) + fm = FileContentsManager(root_dir=root) path = fm._get_os_path('test.ipynb') - fs_path = os.path.join(fm.notebook_dir, 'test.ipynb') + fs_path = os.path.join(fm.root_dir, 'test.ipynb') self.assertEqual(path, fs_path) - fm = FileNotebookManager(notebook_dir=nbdir) + fm = FileContentsManager(root_dir=root) path = fm._get_os_path('test.ipynb', '////') - fs_path = os.path.join(fm.notebook_dir, 'test.ipynb') + fs_path = os.path.join(fm.root_dir, 'test.ipynb') self.assertEqual(path, fs_path) - + def test_checkpoint_subdir(self): subd = u'sub ∂ir' cp_name = 'test-cp.ipynb' with TemporaryDirectory() as td: - nbdir = td + root = td os.mkdir(os.path.join(td, subd)) - fm = FileNotebookManager(notebook_dir=nbdir) + fm = FileContentsManager(root_dir=root) cp_dir = fm.get_checkpoint_path('cp', 'test.ipynb', '/') cp_subdir = fm.get_checkpoint_path('cp', 'test.ipynb', '/%s/' % subd) self.assertNotEqual(cp_dir, cp_subdir) - self.assertEqual(cp_dir, os.path.join(nbdir, fm.checkpoint_dir, cp_name)) - self.assertEqual(cp_subdir, os.path.join(nbdir, subd, fm.checkpoint_dir, cp_name)) - + self.assertEqual(cp_dir, os.path.join(root, fm.checkpoint_dir, cp_name)) + self.assertEqual(cp_subdir, os.path.join(root, subd, fm.checkpoint_dir, cp_name)) + + +class TestContentsManager(TestCase): -class TestNotebookManager(TestCase): - def setUp(self): self._temp_dir = TemporaryDirectory() self.td = self._temp_dir.name - self.notebook_manager = FileNotebookManager( - notebook_dir=self.td, + self.contents_manager = FileContentsManager( + root_dir=self.td, log=logging.getLogger() ) - + def tearDown(self): self._temp_dir.cleanup() - + def make_dir(self, abs_path, rel_path): """make subdirectory, rel_path is the relative path to that directory from the location where the server started""" @@ -91,31 +91,31 @@ class TestNotebookManager(TestCase): os.makedirs(os_path) except OSError: print("Directory already exists: %r" % os_path) - + def add_code_cell(self, nb): output = current.new_output("display_data", output_javascript="alert('hi');") cell = current.new_code_cell("print('hi')", outputs=[output]) if not nb.worksheets: nb.worksheets.append(current.new_worksheet()) nb.worksheets[0].cells.append(cell) - + def new_notebook(self): - nbm = self.notebook_manager - model = nbm.create_notebook() + cm = self.contents_manager + model = cm.create_file() name = model['name'] path = model['path'] - - full_model = nbm.get_notebook(name, path) + + full_model = cm.get_model(name, path) nb = full_model['content'] self.add_code_cell(nb) - - nbm.save_notebook(full_model, name, path) + + cm.save(full_model, name, path) return nb, name, path - - def test_create_notebook(self): - nm = self.notebook_manager + + def test_create_file(self): + cm = self.contents_manager # Test in root directory - model = nm.create_notebook() + model = cm.create_file() assert isinstance(model, dict) self.assertIn('name', model) self.assertIn('path', model) @@ -124,23 +124,23 @@ class TestNotebookManager(TestCase): # Test in sub-directory sub_dir = '/foo/' - self.make_dir(nm.notebook_dir, 'foo') - model = nm.create_notebook(None, sub_dir) + self.make_dir(cm.root_dir, 'foo') + model = cm.create_file(None, sub_dir) assert isinstance(model, dict) self.assertIn('name', model) self.assertIn('path', model) self.assertEqual(model['name'], 'Untitled0.ipynb') self.assertEqual(model['path'], sub_dir.strip('/')) - def test_get_notebook(self): - nm = self.notebook_manager + def test_get(self): + cm = self.contents_manager # Create a notebook - model = nm.create_notebook() + model = cm.create_file() name = model['name'] path = model['path'] # Check that we 'get' on the notebook we just created - model2 = nm.get_notebook(name, path) + model2 = cm.get_model(name, path) assert isinstance(model2, dict) self.assertIn('name', model2) self.assertIn('path', model2) @@ -149,66 +149,66 @@ class TestNotebookManager(TestCase): # Test in sub-directory sub_dir = '/foo/' - self.make_dir(nm.notebook_dir, 'foo') - model = nm.create_notebook(None, sub_dir) - model2 = nm.get_notebook(name, sub_dir) + self.make_dir(cm.root_dir, 'foo') + model = cm.create_file(None, sub_dir) + model2 = cm.get_model(name, sub_dir) assert isinstance(model2, dict) self.assertIn('name', model2) self.assertIn('path', model2) self.assertIn('content', model2) self.assertEqual(model2['name'], 'Untitled0.ipynb') self.assertEqual(model2['path'], sub_dir.strip('/')) - - def test_update_notebook(self): - nm = self.notebook_manager + + def test_update(self): + cm = self.contents_manager # Create a notebook - model = nm.create_notebook() + model = cm.create_file() name = model['name'] path = model['path'] # Change the name in the model for rename model['name'] = 'test.ipynb' - model = nm.update_notebook(model, name, path) + model = cm.update(model, name, path) assert isinstance(model, dict) self.assertIn('name', model) self.assertIn('path', model) self.assertEqual(model['name'], 'test.ipynb') # Make sure the old name is gone - self.assertRaises(HTTPError, nm.get_notebook, name, path) + self.assertRaises(HTTPError, cm.get_model, name, path) # Test in sub-directory # Create a directory and notebook in that directory sub_dir = '/foo/' - self.make_dir(nm.notebook_dir, 'foo') - model = nm.create_notebook(None, sub_dir) + self.make_dir(cm.root_dir, 'foo') + model = cm.create_file(None, sub_dir) name = model['name'] path = model['path'] - + # Change the name in the model for rename model['name'] = 'test_in_sub.ipynb' - model = nm.update_notebook(model, name, path) + model = cm.update(model, name, path) assert isinstance(model, dict) self.assertIn('name', model) self.assertIn('path', model) self.assertEqual(model['name'], 'test_in_sub.ipynb') self.assertEqual(model['path'], sub_dir.strip('/')) - + # Make sure the old name is gone - self.assertRaises(HTTPError, nm.get_notebook, name, path) + self.assertRaises(HTTPError, cm.get_model, name, path) - def test_save_notebook(self): - nm = self.notebook_manager + def test_save(self): + cm = self.contents_manager # Create a notebook - model = nm.create_notebook() + model = cm.create_file() name = model['name'] path = model['path'] # Get the model with 'content' - full_model = nm.get_notebook(name, path) + full_model = cm.get_model(name, path) # Save the notebook - model = nm.save_notebook(full_model, name, path) + model = cm.save(full_model, name, path) assert isinstance(model, dict) self.assertIn('name', model) self.assertIn('path', model) @@ -218,103 +218,84 @@ class TestNotebookManager(TestCase): # Test in sub-directory # Create a directory and notebook in that directory sub_dir = '/foo/' - self.make_dir(nm.notebook_dir, 'foo') - model = nm.create_notebook(None, sub_dir) + self.make_dir(cm.root_dir, 'foo') + model = cm.create_file(None, sub_dir) name = model['name'] path = model['path'] - model = nm.get_notebook(name, path) + model = cm.get_model(name, path) # Change the name in the model for rename - model = nm.save_notebook(model, name, path) + model = cm.save(model, name, path) assert isinstance(model, dict) self.assertIn('name', model) self.assertIn('path', model) self.assertEqual(model['name'], 'Untitled0.ipynb') self.assertEqual(model['path'], sub_dir.strip('/')) - def test_save_notebook_with_script(self): - nm = self.notebook_manager - # Create a notebook - model = nm.create_notebook() - nm.save_script = True - model = nm.create_notebook() - name = model['name'] - path = model['path'] - - # Get the model with 'content' - full_model = nm.get_notebook(name, path) - - # Save the notebook - model = nm.save_notebook(full_model, name, path) - - # Check that the script was created - py_path = os.path.join(nm.notebook_dir, os.path.splitext(name)[0]+'.py') - assert os.path.exists(py_path), py_path - - def test_delete_notebook(self): - nm = self.notebook_manager + def test_delete(self): + cm = self.contents_manager # Create a notebook nb, name, path = self.new_notebook() - + # Delete the notebook - nm.delete_notebook(name, path) - + cm.delete(name, path) + # Check that a 'get' on the deleted notebook raises and error - self.assertRaises(HTTPError, nm.get_notebook, name, path) - - def test_copy_notebook(self): - nm = self.notebook_manager + self.assertRaises(HTTPError, cm.get_model, name, path) + + def test_copy(self): + cm = self.contents_manager path = u'å b' name = u'nb √.ipynb' - os.mkdir(os.path.join(nm.notebook_dir, path)) - orig = nm.create_notebook({'name' : name}, path=path) - + os.mkdir(os.path.join(cm.root_dir, path)) + orig = cm.create_file({'name' : name}, path=path) + # copy with unspecified name - copy = nm.copy_notebook(name, path=path) + copy = cm.copy(name, path=path) self.assertEqual(copy['name'], orig['name'].replace('.ipynb', '-Copy0.ipynb')) - + # copy with specified name - copy2 = nm.copy_notebook(name, u'copy 2.ipynb', path=path) + copy2 = cm.copy(name, u'copy 2.ipynb', path=path) self.assertEqual(copy2['name'], u'copy 2.ipynb') - + def test_trust_notebook(self): - nbm = self.notebook_manager + cm = self.contents_manager nb, name, path = self.new_notebook() - - untrusted = nbm.get_notebook(name, path)['content'] - assert not nbm.notary.check_cells(untrusted) - + + untrusted = cm.get_model(name, path)['content'] + assert not cm.notary.check_cells(untrusted) + # print(untrusted) - nbm.trust_notebook(name, path) - trusted = nbm.get_notebook(name, path)['content'] + cm.trust_notebook(name, path) + trusted = cm.get_model(name, path)['content'] # print(trusted) - assert nbm.notary.check_cells(trusted) - + assert cm.notary.check_cells(trusted) + def test_mark_trusted_cells(self): - nbm = self.notebook_manager + cm = self.contents_manager nb, name, path = self.new_notebook() - - nbm.mark_trusted_cells(nb, name, path) + + cm.mark_trusted_cells(nb, name, path) for cell in nb.worksheets[0].cells: if cell.cell_type == 'code': assert not cell.trusted - - nbm.trust_notebook(name, path) - nb = nbm.get_notebook(name, path)['content'] + + cm.trust_notebook(name, path) + nb = cm.get_model(name, path)['content'] for cell in nb.worksheets[0].cells: if cell.cell_type == 'code': assert cell.trusted def test_check_and_sign(self): - nbm = self.notebook_manager + cm = self.contents_manager nb, name, path = self.new_notebook() - - nbm.mark_trusted_cells(nb, name, path) - nbm.check_and_sign(nb, name, path) - assert not nbm.notary.check_signature(nb) - - nbm.trust_notebook(name, path) - nb = nbm.get_notebook(name, path)['content'] - nbm.mark_trusted_cells(nb, name, path) - nbm.check_and_sign(nb, name, path) - assert nbm.notary.check_signature(nb) + + cm.mark_trusted_cells(nb, name, path) + cm.check_and_sign(nb, name, path) + assert not cm.notary.check_signature(nb) + + cm.trust_notebook(name, path) + nb = cm.get_model(name, path)['content'] + cm.mark_trusted_cells(nb, name, path) + cm.check_and_sign(nb, name, path) + assert cm.notary.check_signature(nb) diff --git a/IPython/html/services/kernels/handlers.py b/IPython/html/services/kernels/handlers.py index 58c2355..b51861e 100644 --- a/IPython/html/services/kernels/handlers.py +++ b/IPython/html/services/kernels/handlers.py @@ -27,8 +27,16 @@ class MainKernelHandler(IPythonHandler): @web.authenticated @json_errors def post(self): + model = self.get_json_body() + if model is None: + raise web.HTTPError(400, "No JSON data provided") + try: + name = model['name'] + except KeyError: + raise web.HTTPError(400, "Missing field in JSON data: name") + km = self.kernel_manager - kernel_id = km.start_kernel() + kernel_id = km.start_kernel(kernel_name=name) model = km.kernel_model(kernel_id) location = url_path_join(self.base_url, 'api', 'kernels', kernel_id) self.set_header('Location', url_escape(location)) @@ -76,6 +84,9 @@ class KernelActionHandler(IPythonHandler): class ZMQChannelHandler(AuthenticatedZMQStreamHandler): + def __repr__(self): + return "%s(%s)" % (self.__class__.__name__, getattr(self, 'kernel_id', 'uninitialized')) + def create_stream(self): km = self.kernel_manager meth = getattr(km, 'connect_%s' % self.channel) @@ -137,6 +148,12 @@ class ZMQChannelHandler(AuthenticatedZMQStreamHandler): self.zmq_stream.on_recv(self._on_zmq_reply) def on_message(self, msg): + if self.zmq_stream is None: + return + elif self.zmq_stream.closed(): + self.log.info("%s closed, closing websocket.", self) + self.close() + return msg = json.loads(msg) self.session.send(self.zmq_stream, msg) diff --git a/IPython/html/services/kernels/kernelmanager.py b/IPython/html/services/kernels/kernelmanager.py index ff27b59..3132efb 100644 --- a/IPython/html/services/kernels/kernelmanager.py +++ b/IPython/html/services/kernels/kernelmanager.py @@ -72,8 +72,8 @@ class MappingKernelManager(MultiKernelManager): os_path = os.path.dirname(os_path) return os_path - def start_kernel(self, kernel_id=None, path=None, **kwargs): - """Start a kernel for a session an return its kernel_id. + def start_kernel(self, kernel_id=None, path=None, kernel_name='python', **kwargs): + """Start a kernel for a session and return its kernel_id. Parameters ---------- @@ -84,12 +84,16 @@ class MappingKernelManager(MultiKernelManager): path : API path The API path (unicode, '/' delimited) for the cwd. Will be transformed to an OS path relative to root_dir. + kernel_name : str + The name identifying which kernel spec to launch. This is ignored if + an existing kernel is returned, but it may be checked in the future. """ if kernel_id is None: kwargs['extra_arguments'] = self.kernel_argv if path is not None: kwargs['cwd'] = self.cwd_for_path(path) - kernel_id = super(MappingKernelManager, self).start_kernel(**kwargs) + kernel_id = super(MappingKernelManager, self).start_kernel( + kernel_name=kernel_name, **kwargs) self.log.info("Kernel started: %s" % kernel_id) self.log.debug("Kernel args: %r" % kwargs) # register callback for failed auto-restart @@ -111,7 +115,8 @@ class MappingKernelManager(MultiKernelManager): """Return a dictionary of kernel information described in the JSON standard model.""" self._check_kernel_id(kernel_id) - model = {"id":kernel_id} + model = {"id":kernel_id, + "name": self._kernels[kernel_id].kernel_name} return model def list_kernels(self): diff --git a/IPython/html/services/kernels/tests/test_kernels_api.py b/IPython/html/services/kernels/tests/test_kernels_api.py index 5e624a7..c3e3c97 100644 --- a/IPython/html/services/kernels/tests/test_kernels_api.py +++ b/IPython/html/services/kernels/tests/test_kernels_api.py @@ -1,6 +1,6 @@ """Test the kernels service API.""" - +import json import requests from IPython.html.utils import url_path_join @@ -30,8 +30,9 @@ class KernelAPI(object): def get(self, id): return self._req('GET', id) - def start(self): - return self._req('POST', '') + def start(self, name='python'): + body = json.dumps({'name': name}) + return self._req('POST', '', body) def shutdown(self, id): return self._req('DELETE', id) @@ -64,11 +65,14 @@ class KernelAPITest(NotebookTestBase): self.assertEqual(r.status_code, 201) self.assertIsInstance(kern1, dict) + self.assertEqual(r.headers['x-frame-options'], "SAMEORIGIN") + # GET request r = self.kern_api.list() self.assertEqual(r.status_code, 200) assert isinstance(r.json(), list) self.assertEqual(r.json()[0]['id'], kern1['id']) + self.assertEqual(r.json()[0]['name'], kern1['name']) # create another kernel and check that they both are added to the # list of kernels from a GET request @@ -89,6 +93,7 @@ class KernelAPITest(NotebookTestBase): self.assertEqual(r.headers['Location'], '/api/kernels/'+kern2['id']) rekern = r.json() self.assertEqual(rekern['id'], kern2['id']) + self.assertEqual(rekern['name'], kern2['name']) def test_kernel_handler(self): # GET kernel with given id diff --git a/IPython/html/services/kernelspecs/handlers.py b/IPython/html/services/kernelspecs/handlers.py index dbe8382..6561b0b 100644 --- a/IPython/html/services/kernelspecs/handlers.py +++ b/IPython/html/services/kernelspecs/handlers.py @@ -7,6 +7,8 @@ from tornado import web from ...base.handlers import IPythonHandler, json_errors +from IPython.kernel.kernelspec import _pythonfirst + class MainKernelSpecHandler(IPythonHandler): SUPPORTED_METHODS = ('GET',) @@ -16,7 +18,7 @@ class MainKernelSpecHandler(IPythonHandler): def get(self): ksm = self.kernel_spec_manager results = [] - for kernel_name in ksm.find_kernel_specs(): + for kernel_name in sorted(ksm.find_kernel_specs(), key=_pythonfirst): d = ksm.get_kernel_spec(kernel_name).to_dict() d['name'] = kernel_name results.append(d) diff --git a/IPython/html/services/notebooks/filenbmanager.py b/IPython/html/services/notebooks/filenbmanager.py deleted file mode 100644 index b9bd389..0000000 --- a/IPython/html/services/notebooks/filenbmanager.py +++ /dev/null @@ -1,470 +0,0 @@ -"""A notebook manager that uses the local file system for storage.""" - -# Copyright (c) IPython Development Team. -# Distributed under the terms of the Modified BSD License. - -import io -import os -import glob -import shutil - -from tornado import web - -from .nbmanager import NotebookManager -from IPython.nbformat import current -from IPython.utils.path import ensure_dir_exists -from IPython.utils.traitlets import Unicode, Bool, TraitError -from IPython.utils.py3compat import getcwd -from IPython.utils import tz -from IPython.html.utils import is_hidden, to_os_path - -def sort_key(item): - """Case-insensitive sorting.""" - return item['name'].lower() - -#----------------------------------------------------------------------------- -# Classes -#----------------------------------------------------------------------------- - -class FileNotebookManager(NotebookManager): - - save_script = Bool(False, config=True, - help="""Automatically create a Python script when saving the notebook. - - For easier use of import, %run and %load across notebooks, a - .py script will be created next to any - .ipynb on each save. This can also be set with the - short `--script` flag. - """ - ) - notebook_dir = Unicode(getcwd(), config=True) - - def _notebook_dir_changed(self, name, old, new): - """Do a bit of validation of the notebook dir.""" - if not os.path.isabs(new): - # If we receive a non-absolute path, make it absolute. - self.notebook_dir = os.path.abspath(new) - return - if not os.path.exists(new) or not os.path.isdir(new): - raise TraitError("notebook dir %r is not a directory" % new) - - checkpoint_dir = Unicode('.ipynb_checkpoints', config=True, - help="""The directory name in which to keep notebook checkpoints - - This is a path relative to the notebook's own directory. - - By default, it is .ipynb_checkpoints - """ - ) - - def _copy(self, src, dest): - """copy src to dest - - like shutil.copy2, but log errors in copystat - """ - shutil.copyfile(src, dest) - try: - shutil.copystat(src, dest) - except OSError as e: - self.log.debug("copystat on %s failed", dest, exc_info=True) - - def get_notebook_names(self, path=''): - """List all notebook names in the notebook dir and path.""" - path = path.strip('/') - if not os.path.isdir(self._get_os_path(path=path)): - raise web.HTTPError(404, 'Directory not found: ' + path) - names = glob.glob(self._get_os_path('*'+self.filename_ext, path)) - names = [os.path.basename(name) - for name in names] - return names - - def path_exists(self, path): - """Does the API-style path (directory) actually exist? - - Parameters - ---------- - path : string - The path to check. This is an API path (`/` separated, - relative to base notebook-dir). - - Returns - ------- - exists : bool - Whether the path is indeed a directory. - """ - path = path.strip('/') - os_path = self._get_os_path(path=path) - return os.path.isdir(os_path) - - def is_hidden(self, path): - """Does the API style path correspond to a hidden directory or file? - - Parameters - ---------- - path : string - The path to check. This is an API path (`/` separated, - relative to base notebook-dir). - - Returns - ------- - exists : bool - Whether the path is hidden. - - """ - path = path.strip('/') - os_path = self._get_os_path(path=path) - return is_hidden(os_path, self.notebook_dir) - - def _get_os_path(self, name=None, path=''): - """Given a notebook name and a URL path, return its file system - path. - - Parameters - ---------- - name : string - The name of a notebook file with the .ipynb extension - path : string - The relative URL path (with '/' as separator) to the named - notebook. - - Returns - ------- - path : string - A file system path that combines notebook_dir (location where - server started), the relative path, and the filename with the - current operating system's url. - """ - if name is not None: - path = path + '/' + name - return to_os_path(path, self.notebook_dir) - - def notebook_exists(self, name, path=''): - """Returns a True if the notebook exists. Else, returns False. - - Parameters - ---------- - name : string - The name of the notebook you are checking. - path : string - The relative path to the notebook (with '/' as separator) - - Returns - ------- - bool - """ - path = path.strip('/') - nbpath = self._get_os_path(name, path=path) - return os.path.isfile(nbpath) - - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def list_dirs(self, path): - """List the directories for a given API style path.""" - path = path.strip('/') - os_path = self._get_os_path('', path) - if not os.path.isdir(os_path): - raise web.HTTPError(404, u'directory does not exist: %r' % os_path) - elif is_hidden(os_path, self.notebook_dir): - self.log.info("Refusing to serve hidden directory, via 404 Error") - raise web.HTTPError(404, u'directory does not exist: %r' % os_path) - dir_names = os.listdir(os_path) - dirs = [] - for name in dir_names: - os_path = self._get_os_path(name, path) - if os.path.isdir(os_path) and not is_hidden(os_path, self.notebook_dir)\ - and self.should_list(name): - try: - model = self.get_dir_model(name, path) - except IOError: - pass - dirs.append(model) - dirs = sorted(dirs, key=sort_key) - return dirs - - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def get_dir_model(self, name, path=''): - """Get the directory model given a directory name and its API style path""" - path = path.strip('/') - os_path = self._get_os_path(name, path) - if not os.path.isdir(os_path): - raise IOError('directory does not exist: %r' % os_path) - info = os.stat(os_path) - last_modified = tz.utcfromtimestamp(info.st_mtime) - created = tz.utcfromtimestamp(info.st_ctime) - # Create the notebook model. - model ={} - model['name'] = name - model['path'] = path - model['last_modified'] = last_modified - model['created'] = created - model['type'] = 'directory' - return model - - def list_notebooks(self, path): - """Returns a list of dictionaries that are the standard model - for all notebooks in the relative 'path'. - - Parameters - ---------- - path : str - the URL path that describes the relative path for the - listed notebooks - - Returns - ------- - notebooks : list of dicts - a list of the notebook models without 'content' - """ - path = path.strip('/') - notebook_names = self.get_notebook_names(path) - notebooks = [self.get_notebook(name, path, content=False) - for name in notebook_names if self.should_list(name)] - notebooks = sorted(notebooks, key=sort_key) - return notebooks - - def get_notebook(self, name, path='', content=True): - """ Takes a path and name for a notebook and returns its model - - Parameters - ---------- - name : str - the name of the notebook - path : str - the URL path that describes the relative path for - the notebook - - Returns - ------- - model : dict - the notebook model. If contents=True, returns the 'contents' - dict in the model as well. - """ - path = path.strip('/') - if not self.notebook_exists(name=name, path=path): - raise web.HTTPError(404, u'Notebook does not exist: %s' % name) - os_path = self._get_os_path(name, path) - info = os.stat(os_path) - last_modified = tz.utcfromtimestamp(info.st_mtime) - created = tz.utcfromtimestamp(info.st_ctime) - # Create the notebook model. - model ={} - model['name'] = name - model['path'] = path - model['last_modified'] = last_modified - model['created'] = created - model['type'] = 'notebook' - if content: - with io.open(os_path, 'r', encoding='utf-8') as f: - try: - nb = current.read(f, u'json') - except Exception as e: - raise web.HTTPError(400, u"Unreadable Notebook: %s %s" % (os_path, e)) - self.mark_trusted_cells(nb, name, path) - model['content'] = nb - return model - - def save_notebook(self, model, name='', path=''): - """Save the notebook model and return the model with no content.""" - path = path.strip('/') - - if 'content' not in model: - raise web.HTTPError(400, u'No notebook JSON data provided') - - # One checkpoint should always exist - if self.notebook_exists(name, path) and not self.list_checkpoints(name, path): - self.create_checkpoint(name, path) - - new_path = model.get('path', path).strip('/') - new_name = model.get('name', name) - - if path != new_path or name != new_name: - self.rename_notebook(name, path, new_name, new_path) - - # Save the notebook file - os_path = self._get_os_path(new_name, new_path) - nb = current.to_notebook_json(model['content']) - - self.check_and_sign(nb, new_name, new_path) - - if 'name' in nb['metadata']: - nb['metadata']['name'] = u'' - try: - self.log.debug("Autosaving notebook %s", os_path) - with io.open(os_path, 'w', encoding='utf-8') as f: - current.write(nb, f, u'json') - except Exception as e: - raise web.HTTPError(400, u'Unexpected error while autosaving notebook: %s %s' % (os_path, e)) - - # Save .py script as well - if self.save_script: - py_path = os.path.splitext(os_path)[0] + '.py' - self.log.debug("Writing script %s", py_path) - try: - with io.open(py_path, 'w', encoding='utf-8') as f: - current.write(nb, f, u'py') - except Exception as e: - raise web.HTTPError(400, u'Unexpected error while saving notebook as script: %s %s' % (py_path, e)) - - model = self.get_notebook(new_name, new_path, content=False) - return model - - def update_notebook(self, model, name, path=''): - """Update the notebook's path and/or name""" - path = path.strip('/') - new_name = model.get('name', name) - new_path = model.get('path', path).strip('/') - if path != new_path or name != new_name: - self.rename_notebook(name, path, new_name, new_path) - model = self.get_notebook(new_name, new_path, content=False) - return model - - def delete_notebook(self, name, path=''): - """Delete notebook by name and path.""" - path = path.strip('/') - os_path = self._get_os_path(name, path) - if not os.path.isfile(os_path): - raise web.HTTPError(404, u'Notebook does not exist: %s' % os_path) - - # clear checkpoints - for checkpoint in self.list_checkpoints(name, path): - checkpoint_id = checkpoint['id'] - cp_path = self.get_checkpoint_path(checkpoint_id, name, path) - if os.path.isfile(cp_path): - self.log.debug("Unlinking checkpoint %s", cp_path) - os.unlink(cp_path) - - self.log.debug("Unlinking notebook %s", os_path) - os.unlink(os_path) - - def rename_notebook(self, old_name, old_path, new_name, new_path): - """Rename a notebook.""" - old_path = old_path.strip('/') - new_path = new_path.strip('/') - if new_name == old_name and new_path == old_path: - return - - new_os_path = self._get_os_path(new_name, new_path) - old_os_path = self._get_os_path(old_name, old_path) - - # Should we proceed with the move? - if os.path.isfile(new_os_path): - raise web.HTTPError(409, u'Notebook with name already exists: %s' % new_os_path) - if self.save_script: - old_py_path = os.path.splitext(old_os_path)[0] + '.py' - new_py_path = os.path.splitext(new_os_path)[0] + '.py' - if os.path.isfile(new_py_path): - raise web.HTTPError(409, u'Python script with name already exists: %s' % new_py_path) - - # Move the notebook file - try: - shutil.move(old_os_path, new_os_path) - except Exception as e: - raise web.HTTPError(500, u'Unknown error renaming notebook: %s %s' % (old_os_path, e)) - - # Move the checkpoints - old_checkpoints = self.list_checkpoints(old_name, old_path) - for cp in old_checkpoints: - checkpoint_id = cp['id'] - old_cp_path = self.get_checkpoint_path(checkpoint_id, old_name, old_path) - new_cp_path = self.get_checkpoint_path(checkpoint_id, new_name, new_path) - if os.path.isfile(old_cp_path): - self.log.debug("Renaming checkpoint %s -> %s", old_cp_path, new_cp_path) - shutil.move(old_cp_path, new_cp_path) - - # Move the .py script - if self.save_script: - shutil.move(old_py_path, new_py_path) - - # Checkpoint-related utilities - - def get_checkpoint_path(self, checkpoint_id, name, path=''): - """find the path to a checkpoint""" - path = path.strip('/') - basename, _ = os.path.splitext(name) - filename = u"{name}-{checkpoint_id}{ext}".format( - name=basename, - checkpoint_id=checkpoint_id, - ext=self.filename_ext, - ) - os_path = self._get_os_path(path=path) - cp_dir = os.path.join(os_path, self.checkpoint_dir) - ensure_dir_exists(cp_dir) - cp_path = os.path.join(cp_dir, filename) - return cp_path - - def get_checkpoint_model(self, checkpoint_id, name, path=''): - """construct the info dict for a given checkpoint""" - path = path.strip('/') - cp_path = self.get_checkpoint_path(checkpoint_id, name, path) - stats = os.stat(cp_path) - last_modified = tz.utcfromtimestamp(stats.st_mtime) - info = dict( - id = checkpoint_id, - last_modified = last_modified, - ) - return info - - # public checkpoint API - - def create_checkpoint(self, name, path=''): - """Create a checkpoint from the current state of a notebook""" - path = path.strip('/') - nb_path = self._get_os_path(name, path) - # only the one checkpoint ID: - checkpoint_id = u"checkpoint" - cp_path = self.get_checkpoint_path(checkpoint_id, name, path) - self.log.debug("creating checkpoint for notebook %s", name) - self._copy(nb_path, cp_path) - - # return the checkpoint info - return self.get_checkpoint_model(checkpoint_id, name, path) - - def list_checkpoints(self, name, path=''): - """list the checkpoints for a given notebook - - This notebook manager currently only supports one checkpoint per notebook. - """ - path = path.strip('/') - checkpoint_id = "checkpoint" - os_path = self.get_checkpoint_path(checkpoint_id, name, path) - if not os.path.exists(os_path): - return [] - else: - return [self.get_checkpoint_model(checkpoint_id, name, path)] - - - def restore_checkpoint(self, checkpoint_id, name, path=''): - """restore a notebook to a checkpointed state""" - path = path.strip('/') - self.log.info("restoring Notebook %s from checkpoint %s", name, checkpoint_id) - nb_path = self._get_os_path(name, path) - cp_path = self.get_checkpoint_path(checkpoint_id, name, path) - if not os.path.isfile(cp_path): - self.log.debug("checkpoint file does not exist: %s", cp_path) - raise web.HTTPError(404, - u'Notebook checkpoint does not exist: %s-%s' % (name, checkpoint_id) - ) - # ensure notebook is readable (never restore from an unreadable notebook) - with io.open(cp_path, 'r', encoding='utf-8') as f: - current.read(f, u'json') - self._copy(cp_path, nb_path) - self.log.debug("copying %s -> %s", cp_path, nb_path) - - def delete_checkpoint(self, checkpoint_id, name, path=''): - """delete a notebook's checkpoint""" - path = path.strip('/') - cp_path = self.get_checkpoint_path(checkpoint_id, name, path) - if not os.path.isfile(cp_path): - raise web.HTTPError(404, - u'Notebook checkpoint does not exist: %s%s-%s' % (path, name, checkpoint_id) - ) - self.log.debug("unlinking %s", cp_path) - os.unlink(cp_path) - - def info_string(self): - return "Serving notebooks from local directory: %s" % self.notebook_dir - - def get_kernel_path(self, name, path='', model=None): - """ Return the path to start kernel in """ - return os.path.join(self.notebook_dir, path) diff --git a/IPython/html/services/notebooks/handlers.py b/IPython/html/services/notebooks/handlers.py deleted file mode 100644 index dab6849..0000000 --- a/IPython/html/services/notebooks/handlers.py +++ /dev/null @@ -1,288 +0,0 @@ -"""Tornado handlers for the notebooks web service. - -Authors: - -* Brian Granger -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -import json - -from tornado import web - -from IPython.html.utils import url_path_join, url_escape -from IPython.utils.jsonutil import date_default - -from IPython.html.base.handlers import (IPythonHandler, json_errors, - notebook_path_regex, path_regex, - notebook_name_regex) - -#----------------------------------------------------------------------------- -# Notebook web service handlers -#----------------------------------------------------------------------------- - - -class NotebookHandler(IPythonHandler): - - SUPPORTED_METHODS = (u'GET', u'PUT', u'PATCH', u'POST', u'DELETE') - - def notebook_location(self, name, path=''): - """Return the full URL location of a notebook based. - - Parameters - ---------- - name : unicode - The base name of the notebook, such as "foo.ipynb". - path : unicode - The URL path of the notebook. - """ - return url_escape(url_path_join( - self.base_url, 'api', 'notebooks', path, name - )) - - def _finish_model(self, model, location=True): - """Finish a JSON request with a model, setting relevant headers, etc.""" - if location: - location = self.notebook_location(model['name'], model['path']) - self.set_header('Location', location) - self.set_header('Last-Modified', model['last_modified']) - self.finish(json.dumps(model, default=date_default)) - - @web.authenticated - @json_errors - def get(self, path='', name=None): - """Return a Notebook or list of notebooks. - - * GET with path and no notebook name lists notebooks in a directory - * GET with path and notebook name returns notebook JSON - """ - nbm = self.notebook_manager - # Check to see if a notebook name was given - if name is None: - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. This should only handle notebooks - # and not directories. - dirs = nbm.list_dirs(path) - notebooks = [] - index = [] - for nb in nbm.list_notebooks(path): - if nb['name'].lower() == 'index.ipynb': - index.append(nb) - else: - notebooks.append(nb) - notebooks = index + dirs + notebooks - self.finish(json.dumps(notebooks, default=date_default)) - return - # get and return notebook representation - model = nbm.get_notebook(name, path) - self._finish_model(model, location=False) - - @web.authenticated - @json_errors - def patch(self, path='', name=None): - """PATCH renames a notebook without re-uploading content.""" - nbm = self.notebook_manager - if name is None: - raise web.HTTPError(400, u'Notebook name missing') - model = self.get_json_body() - if model is None: - raise web.HTTPError(400, u'JSON body missing') - model = nbm.update_notebook(model, name, path) - self._finish_model(model) - - def _copy_notebook(self, copy_from, path, copy_to=None): - """Copy a notebook in path, optionally specifying the new name. - - Only support copying within the same directory. - """ - self.log.info(u"Copying notebook from %s/%s to %s/%s", - path, copy_from, - path, copy_to or '', - ) - model = self.notebook_manager.copy_notebook(copy_from, copy_to, path) - self.set_status(201) - self._finish_model(model) - - def _upload_notebook(self, model, path, name=None): - """Upload a notebook - - If name specified, create it in path/name. - """ - self.log.info(u"Uploading notebook to %s/%s", path, name or '') - if name: - model['name'] = name - - model = self.notebook_manager.create_notebook(model, path) - self.set_status(201) - self._finish_model(model) - - def _create_empty_notebook(self, path, name=None): - """Create an empty notebook in path - - If name specified, create it in path/name. - """ - self.log.info(u"Creating new notebook in %s/%s", path, name or '') - model = {} - if name: - model['name'] = name - model = self.notebook_manager.create_notebook(model, path=path) - self.set_status(201) - self._finish_model(model) - - def _save_notebook(self, model, path, name): - """Save an existing notebook.""" - self.log.info(u"Saving notebook at %s/%s", path, name) - model = self.notebook_manager.save_notebook(model, name, path) - if model['path'] != path.strip('/') or model['name'] != name: - # a rename happened, set Location header - location = True - else: - location = False - self._finish_model(model, location) - - @web.authenticated - @json_errors - def post(self, path='', name=None): - """Create a new notebook in the specified path. - - POST creates new notebooks. The server always decides on the notebook name. - - POST /api/notebooks/path - New untitled notebook in path. If content specified, upload a - notebook, otherwise start empty. - POST /api/notebooks/path?copy=OtherNotebook.ipynb - New copy of OtherNotebook in path - """ - - if name is not None: - raise web.HTTPError(400, "Only POST to directories. Use PUT for full names.") - - model = self.get_json_body() - - if model is not None: - copy_from = model.get('copy_from') - if copy_from: - if model.get('content'): - raise web.HTTPError(400, "Can't upload and copy at the same time.") - self._copy_notebook(copy_from, path) - else: - self._upload_notebook(model, path) - else: - self._create_empty_notebook(path) - - @web.authenticated - @json_errors - def put(self, path='', name=None): - """Saves the notebook in the location specified by name and path. - - PUT is very similar to POST, but the requester specifies the name, - whereas with POST, the server picks the name. - - PUT /api/notebooks/path/Name.ipynb - Save notebook at ``path/Name.ipynb``. Notebook structure is specified - in `content` key of JSON request body. If content is not specified, - create a new empty notebook. - PUT /api/notebooks/path/Name.ipynb?copy=OtherNotebook.ipynb - Copy OtherNotebook to Name - """ - if name is None: - raise web.HTTPError(400, "Only PUT to full names. Use POST for directories.") - - model = self.get_json_body() - if model: - copy_from = model.get('copy_from') - if copy_from: - if model.get('content'): - raise web.HTTPError(400, "Can't upload and copy at the same time.") - self._copy_notebook(copy_from, path, name) - elif self.notebook_manager.notebook_exists(name, path): - self._save_notebook(model, path, name) - else: - self._upload_notebook(model, path, name) - else: - self._create_empty_notebook(path, name) - - @web.authenticated - @json_errors - def delete(self, path='', name=None): - """delete the notebook in the given notebook path""" - nbm = self.notebook_manager - nbm.delete_notebook(name, path) - self.set_status(204) - self.finish() - - -class NotebookCheckpointsHandler(IPythonHandler): - - SUPPORTED_METHODS = ('GET', 'POST') - - @web.authenticated - @json_errors - def get(self, path='', name=None): - """get lists checkpoints for a notebook""" - nbm = self.notebook_manager - checkpoints = nbm.list_checkpoints(name, path) - data = json.dumps(checkpoints, default=date_default) - self.finish(data) - - @web.authenticated - @json_errors - def post(self, path='', name=None): - """post creates a new checkpoint""" - nbm = self.notebook_manager - checkpoint = nbm.create_checkpoint(name, path) - data = json.dumps(checkpoint, default=date_default) - location = url_path_join(self.base_url, 'api/notebooks', - path, name, 'checkpoints', checkpoint['id']) - self.set_header('Location', url_escape(location)) - self.set_status(201) - self.finish(data) - - -class ModifyNotebookCheckpointsHandler(IPythonHandler): - - SUPPORTED_METHODS = ('POST', 'DELETE') - - @web.authenticated - @json_errors - def post(self, path, name, checkpoint_id): - """post restores a notebook from a checkpoint""" - nbm = self.notebook_manager - nbm.restore_checkpoint(checkpoint_id, name, path) - self.set_status(204) - self.finish() - - @web.authenticated - @json_errors - def delete(self, path, name, checkpoint_id): - """delete clears a checkpoint for a given notebook""" - nbm = self.notebook_manager - nbm.delete_checkpoint(checkpoint_id, name, path) - self.set_status(204) - self.finish() - -#----------------------------------------------------------------------------- -# URL to handler mappings -#----------------------------------------------------------------------------- - - -_checkpoint_id_regex = r"(?P[\w-]+)" - -default_handlers = [ - (r"/api/notebooks%s/checkpoints" % notebook_path_regex, NotebookCheckpointsHandler), - (r"/api/notebooks%s/checkpoints/%s" % (notebook_path_regex, _checkpoint_id_regex), - ModifyNotebookCheckpointsHandler), - (r"/api/notebooks%s" % notebook_path_regex, NotebookHandler), - (r"/api/notebooks%s" % path_regex, NotebookHandler), -] - diff --git a/IPython/html/services/notebooks/nbmanager.py b/IPython/html/services/notebooks/nbmanager.py deleted file mode 100644 index d5b6907..0000000 --- a/IPython/html/services/notebooks/nbmanager.py +++ /dev/null @@ -1,287 +0,0 @@ -"""A base class notebook manager. - -Authors: - -* Brian Granger -* Zach Sailer -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -from fnmatch import fnmatch -import itertools -import os - -from IPython.config.configurable import LoggingConfigurable -from IPython.nbformat import current, sign -from IPython.utils.traitlets import Instance, Unicode, List - -#----------------------------------------------------------------------------- -# Classes -#----------------------------------------------------------------------------- - -class NotebookManager(LoggingConfigurable): - - filename_ext = Unicode(u'.ipynb') - - notary = Instance(sign.NotebookNotary) - def _notary_default(self): - return sign.NotebookNotary(parent=self) - - hide_globs = List(Unicode, [u'__pycache__'], config=True, help=""" - Glob patterns to hide in file and directory listings. - """) - - # NotebookManager API part 1: methods that must be - # implemented in subclasses. - - def path_exists(self, path): - """Does the API-style path (directory) actually exist? - - Override this method in subclasses. - - Parameters - ---------- - path : string - The path to check - - Returns - ------- - exists : bool - Whether the path does indeed exist. - """ - raise NotImplementedError - - def is_hidden(self, path): - """Does the API style path correspond to a hidden directory or file? - - Parameters - ---------- - path : string - The path to check. This is an API path (`/` separated, - relative to base notebook-dir). - - Returns - ------- - exists : bool - Whether the path is hidden. - - """ - raise NotImplementedError - - def notebook_exists(self, name, path=''): - """Returns a True if the notebook exists. Else, returns False. - - Parameters - ---------- - name : string - The name of the notebook you are checking. - path : string - The relative path to the notebook (with '/' as separator) - - Returns - ------- - bool - """ - raise NotImplementedError('must be implemented in a subclass') - - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def list_dirs(self, path): - """List the directory models for a given API style path.""" - raise NotImplementedError('must be implemented in a subclass') - - # TODO: Remove this after we create the contents web service and directories are - # no longer listed by the notebook web service. - def get_dir_model(self, name, path=''): - """Get the directory model given a directory name and its API style path. - - The keys in the model should be: - * name - * path - * last_modified - * created - * type='directory' - """ - raise NotImplementedError('must be implemented in a subclass') - - def list_notebooks(self, path=''): - """Return a list of notebook dicts without content. - - This returns a list of dicts, each of the form:: - - dict(notebook_id=notebook,name=name) - - This list of dicts should be sorted by name:: - - data = sorted(data, key=lambda item: item['name']) - """ - raise NotImplementedError('must be implemented in a subclass') - - def get_notebook(self, name, path='', content=True): - """Get the notebook model with or without content.""" - raise NotImplementedError('must be implemented in a subclass') - - def save_notebook(self, model, name, path=''): - """Save the notebook and return the model with no content.""" - raise NotImplementedError('must be implemented in a subclass') - - def update_notebook(self, model, name, path=''): - """Update the notebook and return the model with no content.""" - raise NotImplementedError('must be implemented in a subclass') - - def delete_notebook(self, name, path=''): - """Delete notebook by name and path.""" - raise NotImplementedError('must be implemented in a subclass') - - def create_checkpoint(self, name, path=''): - """Create a checkpoint of the current state of a notebook - - Returns a checkpoint_id for the new checkpoint. - """ - raise NotImplementedError("must be implemented in a subclass") - - def list_checkpoints(self, name, path=''): - """Return a list of checkpoints for a given notebook""" - return [] - - def restore_checkpoint(self, checkpoint_id, name, path=''): - """Restore a notebook from one of its checkpoints""" - raise NotImplementedError("must be implemented in a subclass") - - def delete_checkpoint(self, checkpoint_id, name, path=''): - """delete a checkpoint for a notebook""" - raise NotImplementedError("must be implemented in a subclass") - - def info_string(self): - return "Serving notebooks" - - # NotebookManager API part 2: methods that have useable default - # implementations, but can be overridden in subclasses. - - def get_kernel_path(self, name, path='', model=None): - """ Return the path to start kernel in """ - return path - - def increment_filename(self, basename, path=''): - """Increment a notebook filename without the .ipynb to make it unique. - - Parameters - ---------- - basename : unicode - The name of a notebook without the ``.ipynb`` file extension. - path : unicode - The URL path of the notebooks directory - - Returns - ------- - name : unicode - A notebook name (with the .ipynb extension) that starts - with basename and does not refer to any existing notebook. - """ - path = path.strip('/') - for i in itertools.count(): - name = u'{basename}{i}{ext}'.format(basename=basename, i=i, - ext=self.filename_ext) - if not self.notebook_exists(name, path): - break - return name - - def create_notebook(self, model=None, path=''): - """Create a new notebook and return its model with no content.""" - path = path.strip('/') - if model is None: - model = {} - if 'content' not in model: - metadata = current.new_metadata(name=u'') - model['content'] = current.new_notebook(metadata=metadata) - if 'name' not in model: - model['name'] = self.increment_filename('Untitled', path) - - model['path'] = path - model = self.save_notebook(model, model['name'], model['path']) - return model - - def copy_notebook(self, from_name, to_name=None, path=''): - """Copy an existing notebook and return its new model. - - If to_name not specified, increment `from_name-Copy#.ipynb`. - """ - path = path.strip('/') - model = self.get_notebook(from_name, path) - if not to_name: - base = os.path.splitext(from_name)[0] + '-Copy' - to_name = self.increment_filename(base, path) - model['name'] = to_name - model = self.save_notebook(model, to_name, path) - return model - - def log_info(self): - self.log.info(self.info_string()) - - def trust_notebook(self, name, path=''): - """Explicitly trust a notebook - - Parameters - ---------- - name : string - The filename of the notebook - path : string - The notebook's directory - """ - model = self.get_notebook(name, path) - nb = model['content'] - self.log.warn("Trusting notebook %s/%s", path, name) - self.notary.mark_cells(nb, True) - self.save_notebook(model, name, path) - - def check_and_sign(self, nb, name, path=''): - """Check for trusted cells, and sign the notebook. - - Called as a part of saving notebooks. - - Parameters - ---------- - nb : dict - The notebook structure - name : string - The filename of the notebook - path : string - The notebook's directory - """ - if self.notary.check_cells(nb): - self.notary.sign(nb) - else: - self.log.warn("Saving untrusted notebook %s/%s", path, name) - - def mark_trusted_cells(self, nb, name, path=''): - """Mark cells as trusted if the notebook signature matches. - - Called as a part of loading notebooks. - - Parameters - ---------- - nb : dict - The notebook structure - name : string - The filename of the notebook - path : string - The notebook's directory - """ - trusted = self.notary.check_signature(nb) - if not trusted: - self.log.warn("Notebook %s/%s is not trusted", path, name) - self.notary.mark_cells(nb, trusted) - - def should_list(self, name): - """Should this file/directory name be displayed in a listing?""" - return not any(fnmatch(name, glob) for glob in self.hide_globs) diff --git a/IPython/html/services/sessions/handlers.py b/IPython/html/services/sessions/handlers.py index 7ed47f6..691339f 100644 --- a/IPython/html/services/sessions/handlers.py +++ b/IPython/html/services/sessions/handlers.py @@ -1,20 +1,7 @@ -"""Tornado handlers for the sessions web service. +"""Tornado handlers for the sessions web service.""" -Authors: - -* Zach Sailer -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2013 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. import json @@ -24,10 +11,6 @@ from ...base.handlers import IPythonHandler, json_errors from IPython.utils.jsonutil import date_default from IPython.html.utils import url_path_join, url_escape -#----------------------------------------------------------------------------- -# Session web service handlers -#----------------------------------------------------------------------------- - class SessionRootHandler(IPythonHandler): @@ -45,27 +28,30 @@ class SessionRootHandler(IPythonHandler): # Creates a new session #(unless a session already exists for the named nb) sm = self.session_manager - nbm = self.notebook_manager + cm = self.contents_manager km = self.kernel_manager + model = self.get_json_body() if model is None: raise web.HTTPError(400, "No JSON data provided") try: name = model['notebook']['name'] except KeyError: - raise web.HTTPError(400, "Missing field in JSON data: name") + raise web.HTTPError(400, "Missing field in JSON data: notebook.name") try: path = model['notebook']['path'] except KeyError: - raise web.HTTPError(400, "Missing field in JSON data: path") + raise web.HTTPError(400, "Missing field in JSON data: notebook.path") + try: + kernel_name = model['kernel']['name'] + except KeyError: + raise web.HTTPError(400, "Missing field in JSON data: kernel.name") + # Check to see if session exists if sm.session_exists(name=name, path=path): model = sm.get_session(name=name, path=path) else: - # allow nbm to specify kernels cwd - kernel_path = nbm.get_kernel_path(name=name, path=path) - kernel_id = km.start_kernel(path=kernel_path) - model = sm.create_session(name=name, path=path, kernel_id=kernel_id) + model = sm.create_session(name=name, path=path, kernel_name=kernel_name) location = url_path_join(self.base_url, 'api', 'sessions', model['id']) self.set_header('Location', url_escape(location)) self.set_status(201) @@ -108,10 +94,7 @@ class SessionHandler(IPythonHandler): def delete(self, session_id): # Deletes the session with given session_id sm = self.session_manager - km = self.kernel_manager - session = sm.get_session(session_id=session_id) sm.delete_session(session_id) - km.shutdown_kernel(session['kernel']['id']) self.set_status(204) self.finish() diff --git a/IPython/html/services/sessions/sessionmanager.py b/IPython/html/services/sessions/sessionmanager.py index ec96778..67adbb7 100644 --- a/IPython/html/services/sessions/sessionmanager.py +++ b/IPython/html/services/sessions/sessionmanager.py @@ -23,12 +23,16 @@ from tornado import web from IPython.config.configurable import LoggingConfigurable from IPython.utils.py3compat import unicode_type +from IPython.utils.traitlets import Instance #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- class SessionManager(LoggingConfigurable): + + kernel_manager = Instance('IPython.html.services.kernels.kernelmanager.MappingKernelManager') + contents_manager = Instance('IPython.html.services.contents.manager.ContentsManager', args=()) # Session database initialized below _cursor = None @@ -69,10 +73,15 @@ class SessionManager(LoggingConfigurable): "Create a uuid for a new session" return unicode_type(uuid.uuid4()) - def create_session(self, name=None, path=None, kernel_id=None): + def create_session(self, name=None, path=None, kernel_name='python'): """Creates a session and returns its model""" session_id = self.new_session_id() - return self.save_session(session_id, name=name, path=path, kernel_id=kernel_id) + # allow nbm to specify kernels cwd + kernel_path = self.contents_manager.get_kernel_path(name=name, path=path) + kernel_id = self.kernel_manager.start_kernel(path=kernel_path, + kernel_name=kernel_name) + return self.save_session(session_id, name=name, path=path, + kernel_id=kernel_id) def save_session(self, session_id, name=None, path=None, kernel_id=None): """Saves the items for the session with the given session_id @@ -170,8 +179,7 @@ class SessionManager(LoggingConfigurable): query = "UPDATE session SET %s WHERE session_id=?" % (', '.join(sets)) self.cursor.execute(query, list(kwargs.values()) + [session_id]) - @staticmethod - def row_factory(cursor, row): + def row_factory(self, cursor, row): """Takes sqlite database session row and turns it into a dictionary""" row = sqlite3.Row(cursor, row) model = { @@ -180,9 +188,7 @@ class SessionManager(LoggingConfigurable): 'name': row['name'], 'path': row['path'] }, - 'kernel': { - 'id': row['kernel_id'], - } + 'kernel': self.kernel_manager.kernel_model(row['kernel_id']) } return model @@ -195,5 +201,6 @@ class SessionManager(LoggingConfigurable): def delete_session(self, session_id): """Deletes the row in the session database with given session_id""" # Check that session exists before deleting - self.get_session(session_id=session_id) + session = self.get_session(session_id=session_id) + self.kernel_manager.shutdown_kernel(session['kernel']['id']) self.cursor.execute("DELETE FROM session WHERE session_id=?", (session_id,)) diff --git a/IPython/html/services/sessions/tests/test_sessionmanager.py b/IPython/html/services/sessions/tests/test_sessionmanager.py index d40aa23..ca080e7 100644 --- a/IPython/html/services/sessions/tests/test_sessionmanager.py +++ b/IPython/html/services/sessions/tests/test_sessionmanager.py @@ -5,79 +5,101 @@ from unittest import TestCase from tornado import web from ..sessionmanager import SessionManager +from IPython.html.services.kernels.kernelmanager import MappingKernelManager + +class DummyKernel(object): + def __init__(self, kernel_name='python'): + self.kernel_name = kernel_name + +class DummyMKM(MappingKernelManager): + """MappingKernelManager interface that doesn't start kernels, for testing""" + def __init__(self, *args, **kwargs): + super(DummyMKM, self).__init__(*args, **kwargs) + self.id_letters = iter(u'ABCDEFGHIJK') + + def _new_id(self): + return next(self.id_letters) + + def start_kernel(self, kernel_id=None, path=None, kernel_name='python', **kwargs): + kernel_id = kernel_id or self._new_id() + self._kernels[kernel_id] = DummyKernel(kernel_name=kernel_name) + return kernel_id + + def shutdown_kernel(self, kernel_id, now=False): + del self._kernels[kernel_id] class TestSessionManager(TestCase): def test_get_session(self): - sm = SessionManager() - session_id = sm.new_session_id() - sm.save_session(session_id=session_id, name='test.ipynb', path='/path/to/', kernel_id='5678') + sm = SessionManager(kernel_manager=DummyMKM()) + session_id = sm.create_session(name='test.ipynb', path='/path/to/', + kernel_name='bar')['id'] model = sm.get_session(session_id=session_id) - expected = {'id':session_id, 'notebook':{'name':u'test.ipynb', 'path': u'/path/to/'}, 'kernel':{'id':u'5678'}} + expected = {'id':session_id, + 'notebook':{'name':u'test.ipynb', 'path': u'/path/to/'}, + 'kernel': {'id':u'A', 'name': 'bar'}} self.assertEqual(model, expected) def test_bad_get_session(self): # Should raise error if a bad key is passed to the database. - sm = SessionManager() - session_id = sm.new_session_id() - sm.save_session(session_id=session_id, name='test.ipynb', path='/path/to/', kernel_id='5678') + sm = SessionManager(kernel_manager=DummyMKM()) + session_id = sm.create_session(name='test.ipynb', path='/path/to/', + kernel_name='foo')['id'] self.assertRaises(TypeError, sm.get_session, bad_id=session_id) # Bad keyword def test_list_sessions(self): - sm = SessionManager() - session_id1 = sm.new_session_id() - session_id2 = sm.new_session_id() - session_id3 = sm.new_session_id() - sm.save_session(session_id=session_id1, name='test1.ipynb', path='/path/to/1/', kernel_id='5678') - sm.save_session(session_id=session_id2, name='test2.ipynb', path='/path/to/2/', kernel_id='5678') - sm.save_session(session_id=session_id3, name='test3.ipynb', path='/path/to/3/', kernel_id='5678') + sm = SessionManager(kernel_manager=DummyMKM()) + sessions = [ + sm.create_session(name='test1.ipynb', path='/path/to/1/', kernel_name='python'), + sm.create_session(name='test2.ipynb', path='/path/to/2/', kernel_name='python'), + sm.create_session(name='test3.ipynb', path='/path/to/3/', kernel_name='python'), + ] sessions = sm.list_sessions() - expected = [{'id':session_id1, 'notebook':{'name':u'test1.ipynb', - 'path': u'/path/to/1/'}, 'kernel':{'id':u'5678'}}, - {'id':session_id2, 'notebook': {'name':u'test2.ipynb', - 'path': u'/path/to/2/'}, 'kernel':{'id':u'5678'}}, - {'id':session_id3, 'notebook':{'name':u'test3.ipynb', - 'path': u'/path/to/3/'}, 'kernel':{'id':u'5678'}}] + expected = [{'id':sessions[0]['id'], 'notebook':{'name':u'test1.ipynb', + 'path': u'/path/to/1/'}, 'kernel':{'id':u'A', 'name':'python'}}, + {'id':sessions[1]['id'], 'notebook': {'name':u'test2.ipynb', + 'path': u'/path/to/2/'}, 'kernel':{'id':u'B', 'name':'python'}}, + {'id':sessions[2]['id'], 'notebook':{'name':u'test3.ipynb', + 'path': u'/path/to/3/'}, 'kernel':{'id':u'C', 'name':'python'}}] self.assertEqual(sessions, expected) def test_update_session(self): - sm = SessionManager() - session_id = sm.new_session_id() - sm.save_session(session_id=session_id, name='test.ipynb', path='/path/to/', kernel_id=None) - sm.update_session(session_id, kernel_id='5678') + sm = SessionManager(kernel_manager=DummyMKM()) + session_id = sm.create_session(name='test.ipynb', path='/path/to/', + kernel_name='julia')['id'] sm.update_session(session_id, name='new_name.ipynb') model = sm.get_session(session_id=session_id) - expected = {'id':session_id, 'notebook':{'name':u'new_name.ipynb', 'path': u'/path/to/'}, 'kernel':{'id':u'5678'}} + expected = {'id':session_id, + 'notebook':{'name':u'new_name.ipynb', 'path': u'/path/to/'}, + 'kernel':{'id':u'A', 'name':'julia'}} self.assertEqual(model, expected) def test_bad_update_session(self): # try to update a session with a bad keyword ~ raise error - sm = SessionManager() - session_id = sm.new_session_id() - sm.save_session(session_id=session_id, name='test.ipynb', path='/path/to/', kernel_id='5678') + sm = SessionManager(kernel_manager=DummyMKM()) + session_id = sm.create_session(name='test.ipynb', path='/path/to/', + kernel_name='ir')['id'] self.assertRaises(TypeError, sm.update_session, session_id=session_id, bad_kw='test.ipynb') # Bad keyword def test_delete_session(self): - sm = SessionManager() - session_id1 = sm.new_session_id() - session_id2 = sm.new_session_id() - session_id3 = sm.new_session_id() - sm.save_session(session_id=session_id1, name='test1.ipynb', path='/path/to/1/', kernel_id='5678') - sm.save_session(session_id=session_id2, name='test2.ipynb', path='/path/to/2/', kernel_id='5678') - sm.save_session(session_id=session_id3, name='test3.ipynb', path='/path/to/3/', kernel_id='5678') - sm.delete_session(session_id2) - sessions = sm.list_sessions() - expected = [{'id':session_id1, 'notebook':{'name':u'test1.ipynb', - 'path': u'/path/to/1/'}, 'kernel':{'id':u'5678'}}, - {'id':session_id3, 'notebook':{'name':u'test3.ipynb', - 'path': u'/path/to/3/'}, 'kernel':{'id':u'5678'}}] - self.assertEqual(sessions, expected) + sm = SessionManager(kernel_manager=DummyMKM()) + sessions = [ + sm.create_session(name='test1.ipynb', path='/path/to/1/', kernel_name='python'), + sm.create_session(name='test2.ipynb', path='/path/to/2/', kernel_name='python'), + sm.create_session(name='test3.ipynb', path='/path/to/3/', kernel_name='python'), + ] + sm.delete_session(sessions[1]['id']) + new_sessions = sm.list_sessions() + expected = [{'id':sessions[0]['id'], 'notebook':{'name':u'test1.ipynb', + 'path': u'/path/to/1/'}, 'kernel':{'id':u'A', 'name':'python'}}, + {'id':sessions[2]['id'], 'notebook':{'name':u'test3.ipynb', + 'path': u'/path/to/3/'}, 'kernel':{'id':u'C', 'name':'python'}}] + self.assertEqual(new_sessions, expected) def test_bad_delete_session(self): # try to delete a session that doesn't exist ~ raise error - sm = SessionManager() - session_id = sm.new_session_id() - sm.save_session(session_id=session_id, name='test.ipynb', path='/path/to/', kernel_id='5678') + sm = SessionManager(kernel_manager=DummyMKM()) + sm.create_session(name='test.ipynb', path='/path/to/', kernel_name='python') self.assertRaises(TypeError, sm.delete_session, bad_kwarg='23424') # Bad keyword self.assertRaises(web.HTTPError, sm.delete_session, session_id='23424') # nonexistant diff --git a/IPython/html/services/sessions/tests/test_sessions_api.py b/IPython/html/services/sessions/tests/test_sessions_api.py index 896f4cd..9623415 100644 --- a/IPython/html/services/sessions/tests/test_sessions_api.py +++ b/IPython/html/services/sessions/tests/test_sessions_api.py @@ -37,8 +37,9 @@ class SessionAPI(object): def get(self, id): return self._req('GET', id) - def create(self, name, path): - body = json.dumps({'notebook': {'name':name, 'path':path}}) + def create(self, name, path, kernel_name='python'): + body = json.dumps({'notebook': {'name':name, 'path':path}, + 'kernel': {'name': kernel_name}}) return self._req('POST', '', body) def modify(self, id, name, path): diff --git a/IPython/html/static/auth/js/loginmain.js b/IPython/html/static/auth/js/loginmain.js index d914bf7..a59b3fb 100644 --- a/IPython/html/static/auth/js/loginmain.js +++ b/IPython/html/static/auth/js/loginmain.js @@ -1,21 +1,12 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2008-2011 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. -//============================================================================ -// On document ready -//============================================================================ - - -$(document).ready(function () { - - IPython.page = new IPython.Page(); +var ipython = ipython || {}; +require(['base/js/page'], function(page) { + var page_instance = new page.Page(); $('button#login_submit').addClass("btn btn-default"); - IPython.page.show(); + page_instance.show(); $('input#password_input').focus(); - + + ipython.page = page_instance; }); - diff --git a/IPython/html/static/auth/js/loginwidget.js b/IPython/html/static/auth/js/loginwidget.js index 329ba0e..857caf1 100644 --- a/IPython/html/static/auth/js/loginwidget.js +++ b/IPython/html/static/auth/js/loginwidget.js @@ -1,43 +1,35 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2008-2011 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- - -//============================================================================ -// Login button -//============================================================================ - -var IPython = (function (IPython) { +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. + +define([ + 'base/js/namespace', + 'base/js/utils', + 'jquery', +], function(IPython, utils, $){ "use strict"; var LoginWidget = function (selector, options) { options = options || {}; - this.base_url = options.base_url || IPython.utils.get_body_data("baseUrl"); + this.base_url = options.base_url || utils.get_body_data("baseUrl"); this.selector = selector; if (this.selector !== undefined) { this.element = $(selector); - this.style(); this.bind_events(); } }; - LoginWidget.prototype.style = function () { - this.element.find("button").addClass("btn btn-default btn-sm"); - }; LoginWidget.prototype.bind_events = function () { var that = this; this.element.find("button#logout").click(function () { - window.location = IPython.utils.url_join_encode( + window.location = utils.url_join_encode( that.base_url, "logout" ); }); this.element.find("button#login").click(function () { - window.location = IPython.utils.url_join_encode( + window.location = utils.url_join_encode( that.base_url, "login" ); @@ -47,6 +39,5 @@ var IPython = (function (IPython) { // Set module variables IPython.LoginWidget = LoginWidget; - return IPython; - -}(IPython)); + return {'LoginWidget': LoginWidget}; +}); diff --git a/IPython/html/static/auth/js/logoutmain.js b/IPython/html/static/auth/js/logoutmain.js index df107c6..882d783 100644 --- a/IPython/html/static/auth/js/logoutmain.js +++ b/IPython/html/static/auth/js/logoutmain.js @@ -1,20 +1,10 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2008-2011 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. -//============================================================================ -// On document ready -//============================================================================ - - -$(document).ready(function () { - - IPython.page = new IPython.Page(); - $('#ipython-main-app').addClass('border-box-sizing'); - IPython.page.show(); +var ipython = ipython || {}; +require(['base/js/page'], function(page) { + var page_instance = new page.Page(); + page_instance.show(); + ipython.page = page_instance; }); - diff --git a/IPython/html/static/auth/less/style.less b/IPython/html/static/auth/less/style.less index 42f6f31..4d1919c 100644 --- a/IPython/html/static/auth/less/style.less +++ b/IPython/html/static/auth/less/style.less @@ -1,2 +1,7 @@ +/*! +* +* IPython auth +* +*/ @import "login.less"; @import "logout.less"; \ No newline at end of file diff --git a/IPython/html/static/base/js/dialog.js b/IPython/html/static/base/js/dialog.js index d71c324..c6ffae4 100644 --- a/IPython/html/static/base/js/dialog.js +++ b/IPython/html/static/base/js/dialog.js @@ -1,20 +1,14 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2013 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. -//============================================================================ -// Utility for modal dialogs with bootstrap -//============================================================================ - -IPython.namespace('IPython.dialog'); - -IPython.dialog = (function (IPython) { +define([ + 'base/js/namespace', + 'jquery', +], function(IPython, $) { "use strict"; var modal = function (options) { + var modal = $("
") .addClass("modal") .addClass("fade") @@ -79,26 +73,28 @@ IPython.dialog = (function (IPython) { }); } modal.on("hidden.bs.modal", function () { - if (IPython.notebook) { - var cell = IPython.notebook.get_selected_cell(); + if (options.notebook) { + var cell = options.notebook.get_selected_cell(); if (cell) cell.select(); - IPython.keyboard_manager.enable(); - IPython.keyboard_manager.command_mode(); + } + if (options.keyboard_manager) { + options.keyboard_manager.enable(); + options.keyboard_manager.command_mode(); } }); - if (IPython.keyboard_manager) { - IPython.keyboard_manager.disable(); + if (options.keyboard_manager) { + options.keyboard_manager.disable(); } return modal.modal(options); }; - var edit_metadata = function (md, callback, name) { - name = name || "Cell"; + var edit_metadata = function (options) { + options.name = options.name || "Cell"; var error_div = $('
').css('color', 'red'); var message = - "Manually edit the JSON below to manipulate the metadata for this " + name + "." + + "Manually edit the JSON below to manipulate the metadata for this " + options.name + "." + " We recommend putting custom metadata attributes in an appropriately named sub-structure," + " so they don't conflict with those of others."; @@ -106,7 +102,7 @@ IPython.dialog = (function (IPython) { .attr('rows', '13') .attr('cols', '80') .attr('name', 'metadata') - .text(JSON.stringify(md || {}, null, 2)); + .text(JSON.stringify(options.md || {}, null, 2)); var dialogform = $('
').attr('title', 'Edit the metadata') .append( @@ -128,8 +124,8 @@ IPython.dialog = (function (IPython) { autoIndent: true, mode: 'application/json', }); - var modal = IPython.dialog.modal({ - title: "Edit " + name + " Metadata", + var modal_obj = modal({ + title: "Edit " + options.name + " Metadata", body: dialogform, buttons: { OK: { class : "btn-primary", @@ -143,19 +139,25 @@ IPython.dialog = (function (IPython) { error_div.text('WARNING: Could not save invalid JSON.'); return false; } - callback(new_md); + options.callback(new_md); } }, Cancel: {} - } + }, + notebook: options.notebook, + keyboard_manager: options.keyboard_manager, }); - modal.on('shown.bs.modal', function(){ editor.refresh(); }); + modal_obj.on('shown.bs.modal', function(){ editor.refresh(); }); }; - return { + var dialog = { modal : modal, edit_metadata : edit_metadata, }; -}(IPython)); + // Backwards compatability. + IPython.dialog = dialog; + + return dialog; +}); diff --git a/IPython/html/static/base/js/events.js b/IPython/html/static/base/js/events.js index 3d7d784..48d3183 100644 --- a/IPython/html/static/base/js/events.js +++ b/IPython/html/static/base/js/events.js @@ -1,32 +1,24 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2008-2011 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- - -//============================================================================ -// Events -//============================================================================ +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. // Give us an object to bind all events to. This object should be created // before all other objects so it exists when others register event handlers. -// To trigger an event handler: -// $([IPython.events]).trigger('event.Namespace'); -// To handle it: -// $([IPython.events]).on('event.Namespace',function () {}); +// To register an event handler: +// +// require(['base/js/events'], function (events) { +// events.on("event.Namespace", function () { do_stuff(); }); +// }); -var IPython = (function (IPython) { +define(['base/js/namespace', 'jquery'], function(IPython, $) { "use strict"; - var utils = IPython.utils; - var Events = function () {}; - + + var events = new Events(); + + // Backwards compatability. IPython.Events = Events; - IPython.events = new Events(); - - return IPython; - -}(IPython)); - + IPython.events = events; + + return $([events]); +}); diff --git a/IPython/html/static/base/js/keyboard.js b/IPython/html/static/base/js/keyboard.js index 56391e6..211ce2a 100644 --- a/IPython/html/static/base/js/keyboard.js +++ b/IPython/html/static/base/js/keyboard.js @@ -1,19 +1,14 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2011 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. -//============================================================================ -// Keyboard management -//============================================================================ - -IPython.namespace('IPython.keyboard'); - -IPython.keyboard = (function (IPython) { +define([ + 'base/js/namespace', + 'jquery', + 'base/js/utils', +], function(IPython, $, utils) { "use strict"; + // Setup global keycodes and inverse keycodes. // See http://unixpapa.com/js/key.html for a complete description. The short of @@ -51,8 +46,8 @@ IPython.keyboard = (function (IPython) { '; :': 186, '= +': 187, '- _': 189 }; - var browser = IPython.utils.browser[0]; - var platform = IPython.utils.platform; + var browser = utils.browser[0]; + var platform = utils.platform; if (browser === 'Firefox' || browser === 'Opera' || browser === 'Netscape') { $.extend(_keycodes, _mozilla_keycodes); @@ -130,18 +125,19 @@ IPython.keyboard = (function (IPython) { // Shortcut manager class - var ShortcutManager = function (delay) { + var ShortcutManager = function (delay, events) { this._shortcuts = {}; this._counts = {}; this._timers = {}; this.delay = delay || 800; // delay in milliseconds + this.events = events; }; ShortcutManager.prototype.help = function () { var help = []; for (var shortcut in this._shortcuts) { - var help_string = this._shortcuts[shortcut]['help']; - var help_index = this._shortcuts[shortcut]['help_index']; + var help_string = this._shortcuts[shortcut].help; + var help_index = this._shortcuts[shortcut].help_index; if (help_string) { if (platform === 'MacOS') { shortcut = shortcut.replace('meta', 'cmd'); @@ -182,7 +178,7 @@ IPython.keyboard = (function (IPython) { this._shortcuts[shortcut] = data; if (!suppress_help_update) { // update the keyboard shortcuts notebook help - $([IPython.events]).trigger('rebuild.QuickHelp'); + this.events.trigger('rebuild.QuickHelp'); } }; @@ -191,7 +187,7 @@ IPython.keyboard = (function (IPython) { this.add_shortcut(shortcut, data[shortcut], true); } // update the keyboard shortcuts notebook help - $([IPython.events]).trigger('rebuild.QuickHelp'); + this.events.trigger('rebuild.QuickHelp'); }; ShortcutManager.prototype.remove_shortcut = function (shortcut, suppress_help_update) { @@ -200,7 +196,7 @@ IPython.keyboard = (function (IPython) { delete this._shortcuts[shortcut]; if (!suppress_help_update) { // update the keyboard shortcuts notebook help - $([IPython.events]).trigger('rebuild.QuickHelp'); + this.events.trigger('rebuild.QuickHelp'); } }; @@ -211,7 +207,7 @@ IPython.keyboard = (function (IPython) { var timer = null; if (c[shortcut] === data.count-1) { c[shortcut] = 0; - var timer = t[shortcut]; + timer = t[shortcut]; if (timer) {clearTimeout(timer); delete t[shortcut];} return data.handler(event); } else { @@ -228,7 +224,7 @@ IPython.keyboard = (function (IPython) { var shortcut = event_to_shortcut(event); var data = this._shortcuts[shortcut]; if (data) { - var handler = data['handler']; + var handler = data.handler; if (handler) { if (data.count === 1) { return handler(event); @@ -243,10 +239,10 @@ IPython.keyboard = (function (IPython) { ShortcutManager.prototype.handles = function (event) { var shortcut = event_to_shortcut(event); var data = this._shortcuts[shortcut]; - return !( data === undefined || data.handler === undefined ) - } + return !( data === undefined || data.handler === undefined ); + }; - return { + var keyboard = { keycodes : keycodes, inv_keycodes : inv_keycodes, ShortcutManager : ShortcutManager, @@ -256,4 +252,8 @@ IPython.keyboard = (function (IPython) { event_to_shortcut : event_to_shortcut }; -}(IPython)); + // For backwards compatability. + IPython.keyboard = keyboard; + + return keyboard; +}); diff --git a/IPython/html/static/base/js/namespace.js b/IPython/html/static/base/js/namespace.js index 3b36198..c89cbc7 100644 --- a/IPython/html/static/base/js/namespace.js +++ b/IPython/html/static/base/js/namespace.js @@ -1,34 +1,8 @@ -//---------------------------------------------------------------------------- -// Copyright (C) 2011 The IPython Development Team -// -// Distributed under the terms of the BSD License. The full license is in -// the file COPYING, distributed as part of this software. -//---------------------------------------------------------------------------- +// Copyright (c) IPython Development Team. +// Distributed under the terms of the Modified BSD License. var IPython = IPython || {}; - -IPython.version = "3.0.0-dev"; - -IPython.namespace = function (ns_string) { - "use strict"; - - var parts = ns_string.split('.'), - parent = IPython, - i; - - // String redundant leading global - if (parts[0] === "IPython") { - parts = parts.slice(1); - } - - for (i=0; i