diff --git a/.travis.yml b/.travis.yml index 742511f..fcbc3f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,11 @@ python: - 2.7 - 3.3 before_install: - - pip install jinja2 - easy_install -q pyzmq + - pip install jinja2 sphinx pygments tornado - sudo apt-get install pandoc - - pip install pygments - - pip install sphinx install: - python setup.py install -q script: - - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then iptest -w /tmp; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then iptest3 -w /tmp; fi + - if [[ $TRAVIS_PYTHON_VERSION == '2.'* ]]; then cd /tmp; iptest; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.'* ]]; then cd /tmp; iptest3; fi diff --git a/IPython/__init__.py b/IPython/__init__.py index 55f6f60..b8956fe 100644 --- a/IPython/__init__.py +++ b/IPython/__init__.py @@ -28,8 +28,8 @@ import sys #----------------------------------------------------------------------------- # Don't forget to also update setup.py when this changes! -if sys.version[0:3] < '2.6': - raise ImportError('Python Version 2.6 or above is required for IPython.') +if sys.version_info[:2] < (2,7): + raise ImportError('IPython requires Python Version 2.7 or above.') # Make it easy to import extensions - they are always directly on pythonpath. # Therefore, non-IPython modules can be added to extensions directory. diff --git a/IPython/config/application.py b/IPython/config/application.py index 561269c..18b3659 100644 --- a/IPython/config/application.py +++ b/IPython/config/application.py @@ -142,6 +142,9 @@ class Application(SingletonConfigurable): # The version string of this application. version = Unicode(u'0.0') + + # the argv used to initialize the application + argv = List() # The log level for the application log_level = Enum((0,10,20,30,40,50,'DEBUG','INFO','WARN','ERROR','CRITICAL'), @@ -454,6 +457,7 @@ class Application(SingletonConfigurable): def parse_command_line(self, argv=None): """Parse the command line arguments.""" argv = sys.argv[1:] if argv is None else argv + self.argv = list(argv) if argv and argv[0] == 'help': # turn `ipython help notebook` into `ipython notebook -h` diff --git a/IPython/config/loader.py b/IPython/config/loader.py index 9e2e725..919531a 100644 --- a/IPython/config/loader.py +++ b/IPython/config/loader.py @@ -24,11 +24,11 @@ Authors #----------------------------------------------------------------------------- import __builtin__ as builtin_mod +import argparse import os import re import sys -from IPython.external import argparse from IPython.utils.path import filefind, get_ipython_dir from IPython.utils import py3compat, warn from IPython.utils.encoding import DEFAULT_ENCODING diff --git a/IPython/config/tests/test_application.py b/IPython/config/tests/test_application.py index 4269776..d40939c 100644 --- a/IPython/config/tests/test_application.py +++ b/IPython/config/tests/test_application.py @@ -1,3 +1,4 @@ +# coding: utf-8 """ Tests for IPython.config.application.Application @@ -185,4 +186,8 @@ class TestApplication(TestCase): self.assertEqual(app.bar.b, 5) self.assertEqual(app.extra_args, ['extra', '--disable', 'args']) + def test_unicode_argv(self): + app = MyApp() + app.parse_command_line(['ünîcødé']) + diff --git a/IPython/consoleapp.py b/IPython/consoleapp.py index 26554c4..dddcb8a 100644 --- a/IPython/consoleapp.py +++ b/IPython/consoleapp.py @@ -53,7 +53,7 @@ from IPython.kernel.connect import ConnectionFileMixin # Network Constants #----------------------------------------------------------------------------- -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost #----------------------------------------------------------------------------- # Globals @@ -254,7 +254,7 @@ class IPythonConsoleApp(ConnectionFileMixin): with open(fname) as f: cfg = json.load(f) self.transport = cfg.get('transport', 'tcp') - self.ip = cfg.get('ip', LOCALHOST) + self.ip = cfg.get('ip', localhost()) for channel in ('hb', 'shell', 'iopub', 'stdin', 'control'): name = channel + '_port' @@ -282,7 +282,7 @@ class IPythonConsoleApp(ConnectionFileMixin): if self.sshkey and not self.sshserver: # specifying just the key implies that we are connecting directly self.sshserver = ip - ip = LOCALHOST + ip = localhost() # build connection dict for tunnels: info = dict(ip=ip, @@ -295,7 +295,7 @@ class IPythonConsoleApp(ConnectionFileMixin): self.log.info("Forwarding connections to %s via %s"%(ip, self.sshserver)) # tunnels return a new set of ports, which will be on localhost: - self.ip = LOCALHOST + self.ip = localhost() try: newports = tunnel_to_kernel(info, self.sshserver, self.sshkey) except: diff --git a/IPython/core/alias.py b/IPython/core/alias.py index 0905c19..0140b2e 100644 --- a/IPython/core/alias.py +++ b/IPython/core/alias.py @@ -20,17 +20,15 @@ Authors: # Imports #----------------------------------------------------------------------------- -import __builtin__ -import keyword import os import re import sys from IPython.config.configurable import Configurable -from IPython.core.splitinput import split_user_input +from IPython.core.error import UsageError from IPython.utils.traitlets import List, Instance -from IPython.utils.warn import warn, error +from IPython.utils.warn import error #----------------------------------------------------------------------------- # Utilities @@ -104,6 +102,70 @@ class AliasError(Exception): class InvalidAliasError(AliasError): pass +class Alias(object): + """Callable object storing the details of one alias. + + Instances are registered as magic functions to allow use of aliases. + """ + + # Prepare blacklist + blacklist = {'cd','popd','pushd','dhist','alias','unalias'} + + def __init__(self, shell, name, cmd): + self.shell = shell + self.name = name + self.cmd = cmd + self.nargs = self.validate() + + def validate(self): + """Validate the alias, and return the number of arguments.""" + if self.name in self.blacklist: + raise InvalidAliasError("The name %s can't be aliased " + "because it is a keyword or builtin." % self.name) + try: + caller = self.shell.magics_manager.magics['line'][self.name] + except KeyError: + pass + else: + if not isinstance(caller, Alias): + raise InvalidAliasError("The name %s can't be aliased " + "because it is another magic command." % self.name) + + if not (isinstance(self.cmd, basestring)): + raise InvalidAliasError("An alias command must be a string, " + "got: %r" % self.cmd) + + nargs = self.cmd.count('%s') + + if (nargs > 0) and (self.cmd.find('%l') >= 0): + raise InvalidAliasError('The %s and %l specifiers are mutually ' + 'exclusive in alias definitions.') + + return nargs + + def __repr__(self): + return "".format(self.name, self.cmd) + + def __call__(self, rest=''): + cmd = self.cmd + nargs = self.nargs + # Expand the %l special to be the user's input line + if cmd.find('%l') >= 0: + cmd = cmd.replace('%l', rest) + rest = '' + if nargs==0: + # Simple, argument-less aliases + cmd = '%s %s' % (cmd, rest) + else: + # Handle aliases with positional arguments + args = rest.split(None, nargs) + if len(args) < nargs: + raise UsageError('Alias <%s> requires %s arguments, %s given.' % + (self.name, nargs, len(args))) + cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:])) + + self.shell.system(cmd) + #----------------------------------------------------------------------------- # Main AliasManager class #----------------------------------------------------------------------------- @@ -116,35 +178,19 @@ class AliasManager(Configurable): def __init__(self, shell=None, **kwargs): super(AliasManager, self).__init__(shell=shell, **kwargs) - self.alias_table = {} - self.exclude_aliases() + # For convenient access + self.linemagics = self.shell.magics_manager.magics['line'] self.init_aliases() - def __contains__(self, name): - return name in self.alias_table - - @property - def aliases(self): - return [(item[0], item[1][1]) for item in self.alias_table.iteritems()] - - def exclude_aliases(self): - # set of things NOT to alias (keywords, builtins and some magics) - no_alias = set(['cd','popd','pushd','dhist','alias','unalias']) - no_alias.update(set(keyword.kwlist)) - no_alias.update(set(__builtin__.__dict__.keys())) - self.no_alias = no_alias - def init_aliases(self): - # Load default aliases - for name, cmd in self.default_aliases: - self.soft_define_alias(name, cmd) - - # Load user aliases - for name, cmd in self.user_aliases: + # Load default & user aliases + for name, cmd in self.default_aliases + self.user_aliases: self.soft_define_alias(name, cmd) - def clear_aliases(self): - self.alias_table.clear() + @property + def aliases(self): + return [(n, func.cmd) for (n, func) in self.linemagics.items() + if isinstance(func, Alias)] def soft_define_alias(self, name, cmd): """Define an alias, but don't raise on an AliasError.""" @@ -159,104 +205,33 @@ class AliasManager(Configurable): This will raise an :exc:`AliasError` if there are validation problems. """ - nargs = self.validate_alias(name, cmd) - self.alias_table[name] = (nargs, cmd) + caller = Alias(shell=self.shell, name=name, cmd=cmd) + self.shell.magics_manager.register_function(caller, magic_kind='line', + magic_name=name) - def undefine_alias(self, name): - if name in self.alias_table: - del self.alias_table[name] + def get_alias(self, name): + """Return an alias, or None if no alias by that name exists.""" + aname = self.linemagics.get(name, None) + return aname if isinstance(aname, Alias) else None - def validate_alias(self, name, cmd): - """Validate an alias and return the its number of arguments.""" - if name in self.no_alias: - raise InvalidAliasError("The name %s can't be aliased " - "because it is a keyword or builtin." % name) - if not (isinstance(cmd, basestring)): - raise InvalidAliasError("An alias command must be a string, " - "got: %r" % cmd) - nargs = cmd.count('%s') - if nargs>0 and cmd.find('%l')>=0: - raise InvalidAliasError('The %s and %l specifiers are mutually ' - 'exclusive in alias definitions.') - return nargs + def is_alias(self, name): + """Return whether or not a given name has been defined as an alias""" + return self.get_alias(name) is not None - def call_alias(self, alias, rest=''): - """Call an alias given its name and the rest of the line.""" - cmd = self.transform_alias(alias, rest) - try: - self.shell.system(cmd) - except: - self.shell.showtraceback() - - def transform_alias(self, alias,rest=''): - """Transform alias to system command string.""" - nargs, cmd = self.alias_table[alias] - - if ' ' in cmd and os.path.isfile(cmd): - cmd = '"%s"' % cmd - - # Expand the %l special to be the user's input line - if cmd.find('%l') >= 0: - cmd = cmd.replace('%l', rest) - rest = '' - if nargs==0: - # Simple, argument-less aliases - cmd = '%s %s' % (cmd, rest) + def undefine_alias(self, name): + if self.is_alias(name): + del self.linemagics[name] else: - # Handle aliases with positional arguments - args = rest.split(None, nargs) - if len(args) < nargs: - raise AliasError('Alias <%s> requires %s arguments, %s given.' % - (alias, nargs, len(args))) - cmd = '%s %s' % (cmd % tuple(args[:nargs]),' '.join(args[nargs:])) - return cmd - - def expand_alias(self, line): - """ Expand an alias in the command line + raise ValueError('%s is not an alias' % name) - Returns the provided command line, possibly with the first word - (command) translated according to alias expansion rules. - - [ipython]|16> _ip.expand_aliases("np myfile.txt") - <16> 'q:/opt/np/notepad++.exe myfile.txt' - """ - - pre,_,fn,rest = split_user_input(line) - res = pre + self.expand_aliases(fn, rest) - return res - - def expand_aliases(self, fn, rest): - """Expand multiple levels of aliases: - - if: - - alias foo bar /tmp - alias baz foo - - then: - - baz huhhahhei -> bar /tmp huhhahhei - """ - line = fn + " " + rest - - done = set() - while 1: - pre,_,fn,rest = split_user_input(line, shell_line_split) - if fn in self.alias_table: - if fn in done: - warn("Cyclic alias definition, repeated '%s'" % fn) - return "" - done.add(fn) - - l2 = self.transform_alias(fn, rest) - if l2 == line: - break - # ls -> ls -F should not recurse forever - if l2.split(None,1)[0] == line.split(None,1)[0]: - line = l2 - break - line = l2 - else: - break - - return line + def clear_aliases(self): + for name, cmd in self.aliases: + self.undefine_alias(name) + + def retrieve_alias(self, name): + """Retrieve the command to which an alias expands.""" + caller = self.get_alias(name) + if caller: + return caller.cmd + else: + raise ValueError('%s is not an alias' % name) diff --git a/IPython/core/application.py b/IPython/core/application.py index f2c0c22..cc1ff9d 100644 --- a/IPython/core/application.py +++ b/IPython/core/application.py @@ -53,6 +53,7 @@ from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, Set, Instan # aliases and flags base_aliases = { + 'profile-dir' : 'ProfileDir.location', 'profile' : 'BaseIPythonApplication.profile', 'ipython-dir' : 'BaseIPythonApplication.ipython_dir', 'log-level' : 'Application.log_level', diff --git a/IPython/core/completer.py b/IPython/core/completer.py index 693390a..ee07b26 100644 --- a/IPython/core/completer.py +++ b/IPython/core/completer.py @@ -24,26 +24,26 @@ Tip: to use the tab key as the completion key, call Notes: - Exceptions raised by the completer function are *ignored* (and -generally cause the completion to fail). This is a feature -- since -readline sets the tty device in raw (or cbreak) mode, printing a -traceback wouldn't work well without some complicated hoopla to save, -reset and restore the tty state. + generally cause the completion to fail). This is a feature -- since + readline sets the tty device in raw (or cbreak) mode, printing a + traceback wouldn't work well without some complicated hoopla to save, + reset and restore the tty state. - The evaluation of the NAME.NAME... form may cause arbitrary -application defined code to be executed if an object with a -__getattr__ hook is found. Since it is the responsibility of the -application (or the user) to enable this feature, I consider this an -acceptable risk. More complicated expressions (e.g. function calls or -indexing operations) are *not* evaluated. + application defined code to be executed if an object with a + ``__getattr__`` hook is found. Since it is the responsibility of the + application (or the user) to enable this feature, I consider this an + acceptable risk. More complicated expressions (e.g. function calls or + indexing operations) are *not* evaluated. - GNU readline is also used by the built-in functions input() and -raw_input(), and thus these also benefit/suffer from the completer -features. Clearly an interactive application can benefit by -specifying its own completer function and using raw_input() for all -its input. + raw_input(), and thus these also benefit/suffer from the completer + features. Clearly an interactive application can benefit by + specifying its own completer function and using raw_input() for all + its input. - When the original stdin is not a tty device, GNU readline is never -used, and this module (and the readline module) are silently inactive. + used, and this module (and the readline module) are silently inactive. """ #***************************************************************************** @@ -431,8 +431,7 @@ class IPCompleter(Completer): ) def __init__(self, shell=None, namespace=None, global_namespace=None, - alias_table=None, use_readline=True, - config=None, **kwargs): + use_readline=True, config=None, **kwargs): """IPCompleter() -> completer Return a completer object suitable for use by the readline library @@ -441,17 +440,14 @@ class IPCompleter(Completer): Inputs: - shell: a pointer to the ipython shell itself. This is needed - because this completer knows about magic functions, and those can - only be accessed via the ipython instance. + because this completer knows about magic functions, and those can + only be accessed via the ipython instance. - namespace: an optional dict where completions are performed. - global_namespace: secondary optional dict for completions, to - handle cases (such as IPython embedded inside functions) where - both Python scopes are visible. - - - If alias_table is supplied, it should be a dictionary of aliases - to complete. + handle cases (such as IPython embedded inside functions) where + both Python scopes are visible. use_readline : bool, optional If true, use the readline library. This completer can still function @@ -476,9 +472,6 @@ class IPCompleter(Completer): # List where completion matches will be stored self.matches = [] self.shell = shell - if alias_table is None: - alias_table = {} - self.alias_table = alias_table # Regexp to split filenames with spaces in them self.space_name_re = re.compile(r'([^\\] )') # Hold a local ref. to glob.glob for speed @@ -505,7 +498,6 @@ class IPCompleter(Completer): self.matchers = [self.python_matches, self.file_matches, self.magic_matches, - self.alias_matches, self.python_func_kw_matches, ] @@ -628,22 +620,6 @@ class IPCompleter(Completer): comp += [ pre+m for m in line_magics if m.startswith(bare_text)] return comp - def alias_matches(self, text): - """Match internal system aliases""" - #print 'Completer->alias_matches:',text,'lb',self.text_until_cursor # dbg - - # if we are not in the first 'item', alias matching - # doesn't make sense - unless we are starting with 'sudo' command. - main_text = self.text_until_cursor.lstrip() - if ' ' in main_text and not main_text.startswith('sudo'): - return [] - text = os.path.expanduser(text) - aliases = self.alias_table.keys() - if text == '': - return aliases - else: - return [a for a in aliases if a.startswith(text)] - def python_matches(self,text): """Match attributes or global python names""" diff --git a/IPython/core/debugger.py b/IPython/core/debugger.py index e34a781..9472ec3 100644 --- a/IPython/core/debugger.py +++ b/IPython/core/debugger.py @@ -96,19 +96,24 @@ class Tracer(object): def __init__(self,colors=None): """Create a local debugger instance. - :Parameters: + Parameters + ---------- - - `colors` (None): a string containing the name of the color scheme to - use, it must be one of IPython's valid color schemes. If not given, the - function will default to the current IPython scheme when running inside - IPython, and to 'NoColor' otherwise. + colors : str, optional + The name of the color scheme to use, it must be one of IPython's + valid color schemes. If not given, the function will default to + the current IPython scheme when running inside IPython, and to + 'NoColor' otherwise. - Usage example: + Examples + -------- + :: - from IPython.core.debugger import Tracer; debug_here = Tracer() + from IPython.core.debugger import Tracer; debug_here = Tracer() - ... later in your code - debug_here() # -> will open up the debugger at that point. + Later in your code:: + + debug_here() # -> will open up the debugger at that point. Once the debugger activates, you can use all of its regular commands to step through code, set breakpoints, etc. See the pdb documentation diff --git a/IPython/core/display.py b/IPython/core/display.py index 98f716a..4366c4e 100644 --- a/IPython/core/display.py +++ b/IPython/core/display.py @@ -508,9 +508,9 @@ class Image(DisplayObject): _ACCEPTABLE_EMBEDDINGS = [_FMT_JPEG, _FMT_PNG] def __init__(self, data=None, url=None, filename=None, format=u'png', embed=None, width=None, height=None, retina=False): - """Create a display an PNG/JPEG image given raw data. + """Create a PNG/JPEG image object given raw data. - When this object is returned by an expression or passed to the + When this object is returned by an input cell or passed to the display function, it will result in the image being displayed in the frontend. @@ -657,35 +657,19 @@ class Image(DisplayObject): return unicode(s.split('.')[-1].lower()) -def clear_output(stdout=True, stderr=True, other=True): +def clear_output(wait=False): """Clear the output of the current cell receiving output. - Optionally, each of stdout/stderr or other non-stream data (e.g. anything - produced by display()) can be excluded from the clear event. - - By default, everything is cleared. - Parameters ---------- - stdout : bool [default: True] - Whether to clear stdout. - stderr : bool [default: True] - Whether to clear stderr. - other : bool [default: True] - Whether to clear everything else that is not stdout/stderr - (e.g. figures,images,HTML, any result of display()). - """ + wait : bool [default: false] + Wait to clear the output until new output is available to replace it.""" from IPython.core.interactiveshell import InteractiveShell if InteractiveShell.initialized(): - InteractiveShell.instance().display_pub.clear_output( - stdout=stdout, stderr=stderr, other=other, - ) + InteractiveShell.instance().display_pub.clear_output(wait) else: from IPython.utils import io - if stdout: - print('\033[2K\r', file=io.stdout, end='') - io.stdout.flush() - if stderr: - print('\033[2K\r', file=io.stderr, end='') - io.stderr.flush() - + print('\033[2K\r', file=io.stdout, end='') + io.stdout.flush() + print('\033[2K\r', file=io.stderr, end='') + io.stderr.flush() diff --git a/IPython/core/displaypub.py b/IPython/core/displaypub.py index 254a5d5..fd253ba 100644 --- a/IPython/core/displaypub.py +++ b/IPython/core/displaypub.py @@ -108,14 +108,12 @@ class DisplayPublisher(Configurable): if 'text/plain' in data: print(data['text/plain'], file=io.stdout) - def clear_output(self, stdout=True, stderr=True, other=True): + def clear_output(self, wait=False): """Clear the output of the cell receiving output.""" - if stdout: - print('\033[2K\r', file=io.stdout, end='') - io.stdout.flush() - if stderr: - print('\033[2K\r', file=io.stderr, end='') - io.stderr.flush() + print('\033[2K\r', file=io.stdout, end='') + io.stdout.flush() + print('\033[2K\r', file=io.stderr, end='') + io.stderr.flush() class CapturingDisplayPublisher(DisplayPublisher): @@ -125,8 +123,8 @@ class CapturingDisplayPublisher(DisplayPublisher): def publish(self, source, data, metadata=None): self.outputs.append((source, data, metadata)) - def clear_output(self, stdout=True, stderr=True, other=True): - super(CapturingDisplayPublisher, self).clear_output(stdout, stderr, other) + def clear_output(self, wait=False): + super(CapturingDisplayPublisher, self).clear_output(wait) if other: # empty the list, *do not* reassign a new list del self.outputs[:] diff --git a/IPython/core/fakemodule.py b/IPython/core/fakemodule.py deleted file mode 100644 index 41029f5..0000000 --- a/IPython/core/fakemodule.py +++ /dev/null @@ -1,66 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Class which mimics a module. - -Needed to allow pickle to correctly resolve namespaces during IPython -sessions. -""" - -#***************************************************************************** -# Copyright (C) 2002-2004 Fernando Perez. -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#***************************************************************************** - -import types - -def init_fakemod_dict(fm,adict=None): - """Initialize a FakeModule instance __dict__. - - Kept as a standalone function and not a method so the FakeModule API can - remain basically empty. - - This should be considered for private IPython use, used in managing - namespaces for %run. - - Parameters - ---------- - - fm : FakeModule instance - - adict : dict, optional - """ - - dct = {} - # It seems pydoc (and perhaps others) needs any module instance to - # implement a __nonzero__ method, so we add it if missing: - dct.setdefault('__nonzero__',lambda : True) - dct.setdefault('__file__',__file__) - - if adict is not None: - dct.update(adict) - - # Hard assignment of the object's __dict__. This is nasty but deliberate. - fm.__dict__.clear() - fm.__dict__.update(dct) - - -class FakeModule(types.ModuleType): - """Simple class with attribute access to fake a module. - - This is not meant to replace a module, but to allow inserting a fake - module in sys.modules so that systems which rely on run-time module - importing (like shelve and pickle) work correctly in interactive IPython - sessions. - - Do NOT use this code for anything other than this IPython private hack.""" - - def __init__(self,adict=None): - - # tmp to force __dict__ instance creation, else self.__dict__ fails - self.__iptmp = None - # cleanup our temp trick - del self.__iptmp - # Now, initialize the actual data in the instance dict. - init_fakemod_dict(self,adict) diff --git a/IPython/core/inputtransformer.py b/IPython/core/inputtransformer.py index 1a5f1bb..7582d65 100644 --- a/IPython/core/inputtransformer.py +++ b/IPython/core/inputtransformer.py @@ -278,6 +278,25 @@ _help_end_re = re.compile(r"""(%{0,2} """, re.VERBOSE) +# Extra pseudotokens for multiline strings and data structures +_MULTILINE_STRING = object() +_MULTILINE_STRUCTURE = object() + +def _line_tokens(line): + """Helper for has_comment and ends_in_comment_or_string.""" + readline = StringIO(line).readline + toktypes = set() + try: + for t in generate_tokens(readline): + toktypes.add(t[0]) + except TokenError as e: + # There are only two cases where a TokenError is raised. + if 'multi-line string' in e.args[0]: + toktypes.add(_MULTILINE_STRING) + else: + toktypes.add(_MULTILINE_STRUCTURE) + return toktypes + def has_comment(src): """Indicate whether an input line has (i.e. ends in, or is) a comment. @@ -293,21 +312,31 @@ def has_comment(src): comment : bool True if source has a comment. """ - readline = StringIO(src).readline - toktypes = set() - try: - for t in generate_tokens(readline): - toktypes.add(t[0]) - except TokenError: - pass - return(tokenize2.COMMENT in toktypes) + return (tokenize2.COMMENT in _line_tokens(src)) +def ends_in_comment_or_string(src): + """Indicates whether or not an input line ends in a comment or within + a multiline string. + + Parameters + ---------- + src : string + A single line input string. + + Returns + ------- + comment : bool + True if source ends in a comment or multiline string. + """ + toktypes = _line_tokens(src) + return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes) + @StatelessInputTransformer.wrap def help_end(line): """Translate lines with ?/?? at the end""" m = _help_end_re.search(line) - if m is None or has_comment(line): + if m is None or ends_in_comment_or_string(line): return line target = m.group(1) esc = m.group(3) @@ -359,8 +388,26 @@ def cellmagic(end_on_blank_line=False): line = tpl % (magic_name, first, u'\n'.join(body)) -def _strip_prompts(prompt_re): - """Remove matching input prompts from a block of input.""" +def _strip_prompts(prompt_re, initial_re=None): + """Remove matching input prompts from a block of input. + + Parameters + ---------- + prompt_re : regular expression + A regular expression matching any input prompt (including continuation) + initial_re : regular expression, optional + A regular expression matching only the initial prompt, but not continuation. + If no initial expression is given, prompt_re will be used everywhere. + Used mainly for plain Python prompts, where the continuation prompt + ``...`` is a valid Python expression in Python 3, so shouldn't be stripped. + + If initial_re and prompt_re differ, + only initial_re will be tested against the first line. + If any prompt is found on the first two lines, + prompts will be stripped from the rest of the block. + """ + if initial_re is None: + initial_re = prompt_re line = '' while True: line = (yield line) @@ -368,18 +415,19 @@ def _strip_prompts(prompt_re): # First line of cell if line is None: continue - out, n1 = prompt_re.subn('', line, count=1) + out, n1 = initial_re.subn('', line, count=1) line = (yield out) - # Second line of cell, because people often copy from just after the - # first prompt, so we might not see it in the first line. if line is None: continue + # check for any prompt on the second line of the cell, + # because people often copy from just after the first prompt, + # so we might not see it in the first line. out, n2 = prompt_re.subn('', line, count=1) line = (yield out) if n1 or n2: - # Found the input prompt in the first two lines - check for it in + # Found a prompt in the first two lines - check for it in # the rest of the cell as well. while line is not None: line = (yield prompt_re.sub('', line, count=1)) @@ -394,7 +442,8 @@ def classic_prompt(): """Strip the >>>/... prompts of the Python interactive shell.""" # FIXME: non-capturing version (?:...) usable? prompt_re = re.compile(r'^(>>> ?|\.\.\. ?)') - return _strip_prompts(prompt_re) + initial_re = re.compile(r'^(>>> ?)') + return _strip_prompts(prompt_re, initial_re) @CoroutineInputTransformer.wrap def ipy_prompt(): diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 3453fc6..75f064d 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -47,7 +47,6 @@ from IPython.core.displayhook import DisplayHook from IPython.core.displaypub import DisplayPublisher from IPython.core.error import UsageError from IPython.core.extensions import ExtensionManager -from IPython.core.fakemodule import FakeModule, init_fakemod_dict from IPython.core.formatters import DisplayFormatter from IPython.core.history import HistoryManager from IPython.core.inputsplitter import IPythonInputSplitter, ESC_MAGIC, ESC_MAGIC2 @@ -186,6 +185,13 @@ class ReadlineNoRecord(object): ghi = self.shell.readline.get_history_item return [ghi(x) for x in range(start, end)] + +@undoc +class DummyMod(object): + """A dummy module used for IPython's interactive module when + a namespace must be assigned to the module's __dict__.""" + pass + #----------------------------------------------------------------------------- # Main IPython class #----------------------------------------------------------------------------- @@ -464,7 +470,6 @@ class InteractiveShell(SingletonConfigurable): # because it and init_io have to come after init_readline. self.init_user_ns() self.init_logger() - self.init_alias() self.init_builtins() # The following was in post_config_initialization @@ -496,6 +501,7 @@ class InteractiveShell(SingletonConfigurable): self.init_displayhook() self.init_latextool() self.init_magics() + self.init_alias() self.init_logstart() self.init_pdb() self.init_extension_manager() @@ -819,15 +825,18 @@ class InteractiveShell(SingletonConfigurable): # Things related to the "main" module #------------------------------------------------------------------------- - def new_main_mod(self, filename): + def new_main_mod(self, filename, modname): """Return a new 'main' module object for user code execution. ``filename`` should be the path of the script which will be run in the module. Requests with the same filename will get the same module, with its namespace cleared. + ``modname`` should be the module name - normally either '__main__' or + the basename of the file without the extension. + When scripts are executed via %run, we must keep a reference to their - __main__ module (a FakeModule instance) around so that Python doesn't + __main__ module around so that Python doesn't clear it, rendering references to module globals useless. This method keeps said reference in a private dict, keyed by the @@ -840,9 +849,16 @@ class InteractiveShell(SingletonConfigurable): try: main_mod = self._main_mod_cache[filename] except KeyError: - main_mod = self._main_mod_cache[filename] = FakeModule() + main_mod = self._main_mod_cache[filename] = types.ModuleType(modname, + doc="Module created for script run in IPython") else: - init_fakemod_dict(main_mod) + main_mod.__dict__.clear() + main_mod.__name__ = modname + + main_mod.__file__ = filename + # It seems pydoc (and perhaps others) needs any module instance to + # implement a __nonzero__ method + main_mod.__nonzero__ = lambda : True return main_mod @@ -856,7 +872,7 @@ class InteractiveShell(SingletonConfigurable): In [15]: import IPython - In [16]: m = _ip.new_main_mod(IPython.__file__) + In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython') In [17]: len(_ip._main_mod_cache) > 0 Out[17]: True @@ -900,9 +916,9 @@ class InteractiveShell(SingletonConfigurable): Keywords: - force(False): by default, this routine checks the instance call_pdb - flag and does not actually invoke the debugger if the flag is false. - The 'force' option forces the debugger to activate even if the flag - is false. + flag and does not actually invoke the debugger if the flag is false. + The 'force' option forces the debugger to activate even if the flag + is false. """ if not (force or self.call_pdb): @@ -970,7 +986,7 @@ class InteractiveShell(SingletonConfigurable): # A record of hidden variables we have added to the user namespace, so # we can list later only variables defined in actual interactive use. - self.user_ns_hidden = set() + self.user_ns_hidden = {} # Now that FakeModule produces a real module, we've run into a nasty # problem: after script execution (via %run), the module where the user @@ -1035,9 +1051,6 @@ class InteractiveShell(SingletonConfigurable): """ if user_module is None and user_ns is not None: user_ns.setdefault("__name__", "__main__") - class DummyMod(object): - "A dummy module used for IPython's interactive namespace." - pass user_module = DummyMod() user_module.__dict__ = user_ns @@ -1150,7 +1163,7 @@ class InteractiveShell(SingletonConfigurable): Note that this does not include the displayhook, which also caches objects from the output.""" - return [self.user_ns, self.user_global_ns] + \ + return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \ [m.__dict__ for m in self._main_mod_cache.values()] def reset(self, new_session=True): @@ -1301,7 +1314,8 @@ class InteractiveShell(SingletonConfigurable): # And configure interactive visibility user_ns_hidden = self.user_ns_hidden if interactive: - user_ns_hidden.difference_update(vdict) + for name in vdict: + user_ns_hidden.pop(name, None) else: user_ns_hidden.update(vdict) @@ -1321,7 +1335,7 @@ class InteractiveShell(SingletonConfigurable): for name, obj in variables.iteritems(): if name in self.user_ns and self.user_ns[name] is obj: del self.user_ns[name] - self.user_ns_hidden.discard(name) + self.user_ns_hidden.pop(name, None) #------------------------------------------------------------------------- # Things related to object introspection @@ -1349,9 +1363,7 @@ class InteractiveShell(SingletonConfigurable): namespaces = [ ('Interactive', self.user_ns), ('Interactive (global)', self.user_global_ns), ('Python builtin', builtin_mod.__dict__), - ('Alias', self.alias_manager.alias_table), ] - alias_ns = self.alias_manager.alias_table # initialize results to 'null' found = False; obj = None; ospace = None; ds = None; @@ -1390,8 +1402,6 @@ class InteractiveShell(SingletonConfigurable): # If we finish the for loop (no break), we got all members found = True ospace = nsname - if ns == alias_ns: - isalias = True break # namespace loop # Try to see if it's magic @@ -1926,7 +1936,6 @@ class InteractiveShell(SingletonConfigurable): self.Completer = IPCompleter(shell=self, namespace=self.user_ns, global_namespace=self.user_global_ns, - alias_table=self.alias_manager.alias_table, use_readline=self.has_readline, parent=self, ) @@ -2291,7 +2300,6 @@ class InteractiveShell(SingletonConfigurable): def init_alias(self): self.alias_manager = AliasManager(shell=self, parent=self) self.configurables.append(self.alias_manager) - self.ns_table['alias'] = self.alias_manager.alias_table, #------------------------------------------------------------------------- # Things related to extensions @@ -2973,7 +2981,7 @@ class InteractiveShell(SingletonConfigurable): Optional inputs: - data(None): if data is given, it gets written out to the temp file - immediately, and the file is closed again.""" + immediately, and the file is closed again.""" filename = tempfile.mktemp('.py', prefix) self.tempfiles.append(filename) @@ -3016,13 +3024,14 @@ class InteractiveShell(SingletonConfigurable): Optional Parameters: - raw(False): by default, the processed input is used. If this is - true, the raw input history is used instead. + true, the raw input history is used instead. Note that slices can be called with two notations: N:M -> standard python form, means including items N...(M-1). - N-M -> include items N..M (closed endpoint).""" + N-M -> include items N..M (closed endpoint). + """ lines = self.history_manager.get_range_by_str(range_str, raw=raw) return "\n".join(x for _, _, x in lines) diff --git a/IPython/core/logger.py b/IPython/core/logger.py index d6eff7a..1c21b21 100644 --- a/IPython/core/logger.py +++ b/IPython/core/logger.py @@ -171,11 +171,11 @@ which already exists. But you must first start the logging process with Inputs: - line_mod: possibly modified input, such as the transformations made - by input prefilters or input handlers of various kinds. This should - always be valid Python. + by input prefilters or input handlers of various kinds. This should + always be valid Python. - - line_ori: unmodified input line from the user. This is not - necessarily valid Python. + - line_ori: unmodified input line from the user. This is not + necessarily valid Python. """ # Write the log line, but decide which one according to the diff --git a/IPython/core/magic.py b/IPython/core/magic.py index b21a4fa..65a04fe 100644 --- a/IPython/core/magic.py +++ b/IPython/core/magic.py @@ -489,11 +489,11 @@ class Magics(object): MUST: - Use the method decorators `@line_magic` and `@cell_magic` to decorate - individual methods as magic functions, AND + individual methods as magic functions, AND - Use the class decorator `@magics_class` to ensure that the magic - methods are properly registered at the instance level upon instance - initialization. + methods are properly registered at the instance level upon instance + initialization. See :mod:`magic_functions` for examples of actual implementation classes. """ diff --git a/IPython/core/magic_arguments.py b/IPython/core/magic_arguments.py index 88afb78..e83b521 100644 --- a/IPython/core/magic_arguments.py +++ b/IPython/core/magic_arguments.py @@ -50,9 +50,9 @@ Inheritance diagram: # # The full license is in the file COPYING.txt, distributed with this software. #----------------------------------------------------------------------------- +import argparse # Our own imports -from IPython.external import argparse from IPython.core.error import UsageError from IPython.utils.process import arg_split from IPython.utils.text import dedent diff --git a/IPython/core/magics/execution.py b/IPython/core/magics/execution.py index e01c978..795bbd2 100644 --- a/IPython/core/magics/execution.py +++ b/IPython/core/magics/execution.py @@ -56,6 +56,37 @@ from IPython.utils.warn import warn, error # Magic implementation classes #----------------------------------------------------------------------------- + +class TimeitResult(object): + """ + Object returned by the timeit magic with info about the run. + + Contain the following attributes : + + loops: (int) number of loop done per measurement + repeat: (int) number of time the mesurement has been repeated + best: (float) best execusion time / number + all_runs: (list of float) execusion time of each run (in s) + compile_time: (float) time of statement compilation (s) + + """ + + def __init__(self, loops, repeat, best, all_runs, compile_time, precision): + self.loops = loops + self.repeat = repeat + self.best = best + self.all_runs = all_runs + self.compile_time = compile_time + self._precision = precision + + def _repr_pretty_(self, p , cycle): + unic = u"%d loops, best of %d: %s per loop" % (self.loops, self.repeat, + _format_time(self.best, self._precision)) + p.text(u'') + + + + @magics_class class ExecutionMagics(Magics): """Magics related to code execution, debugging, profiling, etc. @@ -102,75 +133,84 @@ python-profiler package from non-free.""") Options: - -l : you can place restrictions on what or how much of the - profile gets printed. The limit value can be: - - * A string: only information for function names containing this string - is printed. - - * An integer: only these many lines are printed. - - * A float (between 0 and 1): this fraction of the report is printed - (for example, use a limit of 0.4 to see the topmost 40% only). - - You can combine several limits with repeated use of the option. For - example, '-l __init__ -l 5' will print only the topmost 5 lines of - information about class constructors. - - -r: return the pstats.Stats object generated by the profiling. This - object has all the information about the profile in it, and you can - later use it for further analysis or in other functions. - - -s : sort profile by given key. You can provide more than one key - by using the option several times: '-s key1 -s key2 -s key3...'. The - default sorting key is 'time'. - - The following is copied verbatim from the profile documentation - referenced below: - - When more than one key is provided, additional keys are used as - secondary criteria when the there is equality in all keys selected - before them. - - Abbreviations can be used for any key names, as long as the - abbreviation is unambiguous. The following are the keys currently - defined: - - Valid Arg Meaning - "calls" call count - "cumulative" cumulative time - "file" file name - "module" file name - "pcalls" primitive call count - "line" line number - "name" function name - "nfl" name/file/line - "stdname" standard name - "time" internal time - - Note that all sorts on statistics are in descending order (placing - most time consuming items first), where as name, file, and line number - searches are in ascending order (i.e., alphabetical). The subtle - distinction between "nfl" and "stdname" is that the standard name is a - sort of the name as printed, which means that the embedded line - numbers get compared in an odd way. For example, lines 3, 20, and 40 - would (if the file names were the same) appear in the string order - "20" "3" and "40". In contrast, "nfl" does a numeric compare of the - line numbers. In fact, sort_stats("nfl") is the same as - sort_stats("name", "file", "line"). - - -T : save profile results as shown on screen to a text - file. The profile is still shown on screen. - - -D : save (via dump_stats) profile statistics to given - filename. This data is in a format understood by the pstats module, and - is generated by a call to the dump_stats() method of profile - objects. The profile is still shown on screen. - - -q: suppress output to the pager. Best used with -T and/or -D above. + -l + you can place restrictions on what or how much of the + profile gets printed. The limit value can be: + + * A string: only information for function names containing this string + is printed. + + * An integer: only these many lines are printed. + + * A float (between 0 and 1): this fraction of the report is printed + (for example, use a limit of 0.4 to see the topmost 40% only). + + You can combine several limits with repeated use of the option. For + example, ``-l __init__ -l 5`` will print only the topmost 5 lines of + information about class constructors. + + -r + return the pstats.Stats object generated by the profiling. This + object has all the information about the profile in it, and you can + later use it for further analysis or in other functions. + + -s + sort profile by given key. You can provide more than one key + by using the option several times: '-s key1 -s key2 -s key3...'. The + default sorting key is 'time'. + + The following is copied verbatim from the profile documentation + referenced below: + + When more than one key is provided, additional keys are used as + secondary criteria when the there is equality in all keys selected + before them. + + Abbreviations can be used for any key names, as long as the + abbreviation is unambiguous. The following are the keys currently + defined: + + ============ ===================== + Valid Arg Meaning + ============ ===================== + "calls" call count + "cumulative" cumulative time + "file" file name + "module" file name + "pcalls" primitive call count + "line" line number + "name" function name + "nfl" name/file/line + "stdname" standard name + "time" internal time + ============ ===================== + + Note that all sorts on statistics are in descending order (placing + most time consuming items first), where as name, file, and line number + searches are in ascending order (i.e., alphabetical). The subtle + distinction between "nfl" and "stdname" is that the standard name is a + sort of the name as printed, which means that the embedded line + numbers get compared in an odd way. For example, lines 3, 20, and 40 + would (if the file names were the same) appear in the string order + "20" "3" and "40". In contrast, "nfl" does a numeric compare of the + line numbers. In fact, sort_stats("nfl") is the same as + sort_stats("name", "file", "line"). + + -T + save profile results as shown on screen to a text + file. The profile is still shown on screen. + + -D + save (via dump_stats) profile statistics to given + filename. This data is in a format understood by the pstats module, and + is generated by a call to the dump_stats() method of profile + objects. The profile is still shown on screen. + + -q + suppress output to the pager. Best used with -T and/or -D above. If you want to run complete programs under the profiler's control, use - '%run -p [prof_opts] filename.py [args to program]' where prof_opts + ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts contains profiler specific options as described here. You can read the complete documentation for the profile module with:: @@ -362,7 +402,8 @@ python-profiler package from non-free.""") file_finder=get_py_filename): """Run the named file inside IPython as a program. - Usage: + Usage:: + %run [-n -i -e -G] [( -t [-N] | -d [-b] | -p [profile options] )] ( -m mod | file ) [args] @@ -371,14 +412,13 @@ python-profiler package from non-free.""") the program (put in sys.argv). Then, control returns to IPython's prompt. - This is similar to running at a system prompt:\\ - $ python file args\\ + This is similar to running at a system prompt ``python file args``, but with the advantage of giving you IPython's tracebacks, and of loading all variables into your interactive namespace for further use (unless -p is used, see below). The file is executed in a namespace initially consisting only of - __name__=='__main__' and sys.argv constructed as indicated. It thus + ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus sees its environment as if it were being run as a stand-alone program (except for sharing global objects such as previously imported modules). But after execution, the IPython interactive namespace gets @@ -390,33 +430,37 @@ python-profiler package from non-free.""") '*', '?', '[seq]' and '[!seq]' can be used. Additionally, tilde '~' will be expanded into user's home directory. Unlike real shells, quotation does not suppress expansions. Use - *two* back slashes (e.g., '\\\\*') to suppress expansions. + *two* back slashes (e.g. ``\\\\*``) to suppress expansions. To completely disable these expansions, you can use -G flag. Options: - -n: __name__ is NOT set to '__main__', but to the running file's name - without extension (as python does under import). This allows running - scripts and reloading the definitions in them without calling code - protected by an ' if __name__ == "__main__" ' clause. - - -i: run the file in IPython's namespace instead of an empty one. This - is useful if you are experimenting with code written in a text editor - which depends on variables defined interactively. - - -e: ignore sys.exit() calls or SystemExit exceptions in the script - being run. This is particularly useful if IPython is being used to - run unittests, which always exit with a sys.exit() call. In such - cases you are interested in the output of the test results, not in - seeing a traceback of the unittest module. - - -t: print timing information at the end of the run. IPython will give - you an estimated CPU time consumption for your script, which under - Unix uses the resource module to avoid the wraparound problems of - time.clock(). Under Unix, an estimate of time spent on system tasks - is also given (for Windows platforms this is reported as 0.0). - - If -t is given, an additional -N option can be given, where + -n + __name__ is NOT set to '__main__', but to the running file's name + without extension (as python does under import). This allows running + scripts and reloading the definitions in them without calling code + protected by an ``if __name__ == "__main__"`` clause. + + -i + run the file in IPython's namespace instead of an empty one. This + is useful if you are experimenting with code written in a text editor + which depends on variables defined interactively. + + -e + ignore sys.exit() calls or SystemExit exceptions in the script + being run. This is particularly useful if IPython is being used to + run unittests, which always exit with a sys.exit() call. In such + cases you are interested in the output of the test results, not in + seeing a traceback of the unittest module. + + -t + print timing information at the end of the run. IPython will give + you an estimated CPU time consumption for your script, which under + Unix uses the resource module to avoid the wraparound problems of + time.clock(). Under Unix, an estimate of time spent on system tasks + is also given (for Windows platforms this is reported as 0.0). + + If -t is given, an additional ``-N`` option can be given, where must be an integer indicating how many times you want the script to run. The final timing report will include total and per run results. @@ -424,74 +468,78 @@ python-profiler package from non-free.""") In [1]: run -t uniq_stable - IPython CPU timings (estimated):\\ - User : 0.19597 s.\\ - System: 0.0 s.\\ + IPython CPU timings (estimated): + User : 0.19597 s. + System: 0.0 s. In [2]: run -t -N5 uniq_stable - IPython CPU timings (estimated):\\ - Total runs performed: 5\\ - Times : Total Per run\\ - User : 0.910862 s, 0.1821724 s.\\ + IPython CPU timings (estimated): + Total runs performed: 5 + Times : Total Per run + User : 0.910862 s, 0.1821724 s. System: 0.0 s, 0.0 s. - -d: run your program under the control of pdb, the Python debugger. - This allows you to execute your program step by step, watch variables, - etc. Internally, what IPython does is similar to calling: + -d + run your program under the control of pdb, the Python debugger. + This allows you to execute your program step by step, watch variables, + etc. Internally, what IPython does is similar to calling:: - pdb.run('execfile("YOURFILENAME")') + pdb.run('execfile("YOURFILENAME")') - with a breakpoint set on line 1 of your file. You can change the line - number for this automatic breakpoint to be by using the -bN option - (where N must be an integer). For example:: + with a breakpoint set on line 1 of your file. You can change the line + number for this automatic breakpoint to be by using the -bN option + (where N must be an integer). For example:: - %run -d -b40 myscript + %run -d -b40 myscript - will set the first breakpoint at line 40 in myscript.py. Note that - the first breakpoint must be set on a line which actually does - something (not a comment or docstring) for it to stop execution. + will set the first breakpoint at line 40 in myscript.py. Note that + the first breakpoint must be set on a line which actually does + something (not a comment or docstring) for it to stop execution. - Or you can specify a breakpoint in a different file:: + Or you can specify a breakpoint in a different file:: - %run -d -b myotherfile.py:20 myscript + %run -d -b myotherfile.py:20 myscript - When the pdb debugger starts, you will see a (Pdb) prompt. You must - first enter 'c' (without quotes) to start execution up to the first - breakpoint. + When the pdb debugger starts, you will see a (Pdb) prompt. You must + first enter 'c' (without quotes) to start execution up to the first + breakpoint. - Entering 'help' gives information about the use of the debugger. You - can easily see pdb's full documentation with "import pdb;pdb.help()" - at a prompt. + Entering 'help' gives information about the use of the debugger. You + can easily see pdb's full documentation with "import pdb;pdb.help()" + at a prompt. - -p: run program under the control of the Python profiler module (which - prints a detailed report of execution times, function calls, etc). + -p + run program under the control of the Python profiler module (which + prints a detailed report of execution times, function calls, etc). - You can pass other options after -p which affect the behavior of the - profiler itself. See the docs for %prun for details. + You can pass other options after -p which affect the behavior of the + profiler itself. See the docs for %prun for details. - In this mode, the program's variables do NOT propagate back to the - IPython interactive namespace (because they remain in the namespace - where the profiler executes them). + In this mode, the program's variables do NOT propagate back to the + IPython interactive namespace (because they remain in the namespace + where the profiler executes them). - Internally this triggers a call to %prun, see its documentation for - details on the options available specifically for profiling. + Internally this triggers a call to %prun, see its documentation for + details on the options available specifically for profiling. There is one special usage for which the text above doesn't apply: if the filename ends with .ipy, the file is run as ipython script, just as if the commands were written on IPython prompt. - -m: specify module name to load instead of script path. Similar to - the -m option for the python interpreter. Use this option last if you - want to combine with other %run options. Unlike the python interpreter - only source modules are allowed no .pyc or .pyo files. - For example:: + -m + specify module name to load instead of script path. Similar to + the -m option for the python interpreter. Use this option last if you + want to combine with other %run options. Unlike the python interpreter + only source modules are allowed no .pyc or .pyo files. + For example:: - %run -m example + %run -m example - will run the example module. + will run the example module. - -G: disable shell-like glob expansion of arguments. + -G + disable shell-like glob expansion of arguments. """ @@ -550,6 +598,11 @@ python-profiler package from non-free.""") __name__save = self.shell.user_ns['__name__'] prog_ns['__name__'] = '__main__' main_mod = self.shell.user_module + + # Since '%run foo' emulates 'python foo.py' at the cmd line, we must + # set the __file__ global in the script's namespace + # TK: Is this necessary in interactive mode? + prog_ns['__file__'] = filename else: # Run in a fresh, empty namespace if 'n' in opts: @@ -560,13 +613,8 @@ python-profiler package from non-free.""") # The shell MUST hold a reference to prog_ns so after %run # exits, the python deletion mechanism doesn't zero it out # (leaving dangling references). See interactiveshell for details - main_mod = self.shell.new_main_mod(filename) + main_mod = self.shell.new_main_mod(filename, name) prog_ns = main_mod.__dict__ - prog_ns['__name__'] = name - - # Since '%run foo' emulates 'python foo.py' at the cmd line, we must - # set the __file__ global in the script's namespace - prog_ns['__file__'] = filename # pickle fix. See interactiveshell for an explanation. But we need to # make sure that, if we overwrite __main__, we replace it at the end @@ -786,9 +834,9 @@ python-profiler package from non-free.""") """Time execution of a Python statement or expression Usage, in line mode: - %timeit [-n -r [-t|-c]] statement + %timeit [-n -r [-t|-c] -q -p

-o] statement or in cell mode: - %%timeit [-n -r [-t|-c]] setup_code + %%timeit [-n -r [-t|-c] -q -p

-o] setup_code code code... @@ -819,6 +867,11 @@ python-profiler package from non-free.""") -p

: use a precision of

digits to display the timing result. Default: 3 + -q: Quiet, do not print result. + + -o: return a TimeitResult that can be stored in a variable to inspect + the result in more details. + Examples -------- @@ -851,7 +904,7 @@ python-profiler package from non-free.""") import timeit - opts, stmt = self.parse_options(line,'n:r:tcp:', + opts, stmt = self.parse_options(line,'n:r:tcp:qo', posix=False, strict=False) if stmt == "" and cell is None: return @@ -860,6 +913,8 @@ python-profiler package from non-free.""") number = int(getattr(opts, "n", 0)) repeat = int(getattr(opts, "r", timeit.default_repeat)) precision = int(getattr(opts, "p", 3)) + quiet = 'q' in opts + return_result = 'o' in opts if hasattr(opts, "t"): timefunc = time.time if hasattr(opts, "c"): @@ -931,13 +986,15 @@ python-profiler package from non-free.""") if timer.timeit(number) >= 0.2: break number *= 10 - - best = min(timer.repeat(repeat, number)) / number - - print u"%d loops, best of %d: %s per loop" % (number, repeat, - _format_time(best, precision)) - if tc > tc_min: - print "Compiler time: %.2f s" % tc + all_runs = timer.repeat(repeat, number) + best = min(all_runs) / number + if not quiet : + print u"%d loops, best of %d: %s per loop" % (number, repeat, + _format_time(best, precision)) + if tc > tc_min: + print "Compiler time: %.2f s" % tc + if return_result: + return TimeitResult(number, repeat, best, all_runs, tc, precision) @skip_doctest @needs_local_scope diff --git a/IPython/core/magics/logging.py b/IPython/core/magics/logging.py index b1e7b28..3dcb8cb 100644 --- a/IPython/core/magics/logging.py +++ b/IPython/core/magics/logging.py @@ -42,34 +42,48 @@ class LoggingMagics(Magics): history up to that point and then continues logging. %logstart takes a second optional parameter: logging mode. This can be one - of (note that the modes are given unquoted):\\ - append: well, that says it.\\ - backup: rename (if exists) to name~ and start name.\\ - global: single logfile in your home dir, appended to.\\ - over : overwrite existing log.\\ - rotate: create rotating logs name.1~, name.2~, etc. + of (note that the modes are given unquoted): + + append + Keep logging at the end of any existing file. + + backup + Rename any existing file to name~ and start name. + + global + Append to a single logfile in your home directory. + + over + Overwrite any existing log. + + rotate + Create rotating logs: name.1~, name.2~, etc. Options: - -o: log also IPython's output. In this mode, all commands which - generate an Out[NN] prompt are recorded to the logfile, right after - their corresponding input line. The output lines are always - prepended with a '#[Out]# ' marker, so that the log remains valid - Python code. + -o + log also IPython's output. In this mode, all commands which + generate an Out[NN] prompt are recorded to the logfile, right after + their corresponding input line. The output lines are always + prepended with a '#[Out]# ' marker, so that the log remains valid + Python code. Since this marker is always the same, filtering only the output from a log is very easy, using for example a simple awk call:: awk -F'#\\[Out\\]# ' '{if($2) {print $2}}' ipython_log.py - -r: log 'raw' input. Normally, IPython's logs contain the processed - input, so that user lines are logged in their final form, converted - into valid Python. For example, %Exit is logged as - _ip.magic("Exit"). If the -r flag is given, all input is logged - exactly as typed, with no transformations applied. - - -t: put timestamps before each input line logged (these are put in - comments).""" + -r + log 'raw' input. Normally, IPython's logs contain the processed + input, so that user lines are logged in their final form, converted + into valid Python. For example, %Exit is logged as + _ip.magic("Exit"). If the -r flag is given, all input is logged + exactly as typed, with no transformations applied. + + -t + put timestamps before each input line logged (these are put in + comments). + """ opts,par = self.parse_options(parameter_s,'ort') log_output = 'o' in opts diff --git a/IPython/core/magics/namespace.py b/IPython/core/magics/namespace.py index 36f2d02..1d031dc 100644 --- a/IPython/core/magics/namespace.py +++ b/IPython/core/magics/namespace.py @@ -265,9 +265,10 @@ class NamespaceMagics(Magics): user_ns = self.shell.user_ns user_ns_hidden = self.shell.user_ns_hidden + nonmatching = object() # This can never be in user_ns out = [ i for i in user_ns if not i.startswith('_') \ - and not i in user_ns_hidden ] + and (user_ns[i] is not user_ns_hidden.get(i, nonmatching)) ] typelist = parameter_s.split() if typelist: @@ -353,10 +354,10 @@ class NamespaceMagics(Magics): - For {},[],(): their length. - For numpy arrays, a summary with shape, number of - elements, typecode and size in memory. + elements, typecode and size in memory. - Everything else: a string representation, snipping their middle if - too long. + too long. Examples -------- diff --git a/IPython/core/magics/osm.py b/IPython/core/magics/osm.py index 71f0035..3cfc0f0 100644 --- a/IPython/core/magics/osm.py +++ b/IPython/core/magics/osm.py @@ -26,6 +26,7 @@ from pprint import pformat from IPython.core import magic_arguments from IPython.core import oinspect from IPython.core import page +from IPython.core.alias import AliasError, Alias from IPython.core.error import UsageError from IPython.core.magic import ( Magics, compress_dhist, magics_class, line_magic, cell_magic, line_cell_magic @@ -113,10 +114,14 @@ class OSMagics(Magics): # Now try to define a new one try: alias,cmd = par.split(None, 1) - except: - print oinspect.getdoc(self.alias) - else: - self.shell.alias_manager.soft_define_alias(alias, cmd) + except TypeError: + print(oinspect.getdoc(self.alias)) + return + + try: + self.shell.alias_manager.define_alias(alias, cmd) + except AliasError as e: + print(e) # end magic_alias @line_magic @@ -124,7 +129,12 @@ class OSMagics(Magics): """Remove an alias""" aname = parameter_s.strip() - self.shell.alias_manager.undefine_alias(aname) + try: + self.shell.alias_manager.undefine_alias(aname) + except ValueError as e: + print(e) + return + stored = self.shell.db.get('stored_aliases', {} ) if aname in stored: print "Removing %stored alias",aname @@ -182,7 +192,7 @@ class OSMagics(Magics): try: # Removes dots from the name since ipython # will assume names with dots to be python. - if ff not in self.shell.alias_manager: + if not self.shell.alias_manager.is_alias(ff): self.shell.alias_manager.define_alias( ff.replace('.',''), ff) except InvalidAliasError: @@ -190,7 +200,7 @@ class OSMagics(Magics): else: syscmdlist.append(ff) else: - no_alias = self.shell.alias_manager.no_alias + no_alias = Alias.blacklist for pdir in path: os.chdir(pdir) for ff in os.listdir(pdir): diff --git a/IPython/core/oinspect.py b/IPython/core/oinspect.py index 4846861..1dbe761 100644 --- a/IPython/core/oinspect.py +++ b/IPython/core/oinspect.py @@ -156,8 +156,8 @@ def getsource(obj,is_binary=False): Optional inputs: - is_binary: whether the object is known to come from a binary source. - This implementation will skip returning any output for binary objects, but - custom extractors may know how to meaningfully process them.""" + This implementation will skip returning any output for binary objects, but + custom extractors may know how to meaningfully process them.""" if is_binary: return None @@ -545,7 +545,7 @@ class Inspector: - formatter: special formatter for docstrings (see pdoc) - info: a structure with some information fields which may have been - precomputed already. + precomputed already. - detail_level: if set to 1, more information is given. """ @@ -609,7 +609,7 @@ class Inspector: - formatter: special formatter for docstrings (see pdoc) - info: a structure with some information fields which may have been - precomputed already. + precomputed already. - detail_level: if set to 1, more information is given. """ @@ -829,8 +829,8 @@ class Inspector: Arguments: - pattern: string containing shell-like wildcards to use in namespace - searches and optionally a type specification to narrow the search to - objects of that type. + searches and optionally a type specification to narrow the search to + objects of that type. - ns_table: dict of name->namespaces for search. @@ -841,7 +841,7 @@ class Inspector: - ignore_case(False): make the search case-insensitive. - show_all(False): show all names, including those starting with - underscores. + underscores. """ #print 'ps pattern:<%r>' % pattern # dbg diff --git a/IPython/core/page.py b/IPython/core/page.py index a3f7d91..67cde95 100644 --- a/IPython/core/page.py +++ b/IPython/core/page.py @@ -325,9 +325,11 @@ def snip_print(str,width = 75,print_full = 0,header = ''): """Print a string snipping the midsection to fit in width. print_full: mode control: + - 0: only snip long strings - 1: send to page() directly. - 2: snip long strings and ask for full length viewing with page() + Return 1 if snipping was necessary, 0 otherwise.""" if print_full == 1: diff --git a/IPython/core/prefilter.py b/IPython/core/prefilter.py index 49204bf..15aba20 100644 --- a/IPython/core/prefilter.py +++ b/IPython/core/prefilter.py @@ -488,22 +488,6 @@ class AutoMagicChecker(PrefilterChecker): return self.prefilter_manager.get_handler_by_name('magic') -class AliasChecker(PrefilterChecker): - - priority = Integer(800, config=True) - - def check(self, line_info): - "Check if the initital identifier on the line is an alias." - # Note: aliases can not contain '.' - head = line_info.ifun.split('.',1)[0] - if line_info.ifun not in self.shell.alias_manager \ - or head not in self.shell.alias_manager \ - or is_shadowed(head, self.shell): - return None - - return self.prefilter_manager.get_handler_by_name('alias') - - class PythonOpsChecker(PrefilterChecker): priority = Integer(900, config=True) @@ -591,20 +575,6 @@ class PrefilterHandler(Configurable): return "<%s(name=%s)>" % (self.__class__.__name__, self.handler_name) -class AliasHandler(PrefilterHandler): - - handler_name = Unicode('alias') - - def handle(self, line_info): - """Handle alias input lines. """ - transformed = self.shell.alias_manager.expand_aliases(line_info.ifun,line_info.the_rest) - # pre is needed, because it carries the leading whitespace. Otherwise - # aliases won't work in indented sections. - line_out = '%sget_ipython().system(%r)' % (line_info.pre_whitespace, transformed) - - return line_out - - class MacroHandler(PrefilterHandler): handler_name = Unicode("macro") @@ -730,14 +700,12 @@ _default_checkers = [ IPyAutocallChecker, AssignmentChecker, AutoMagicChecker, - AliasChecker, PythonOpsChecker, AutocallChecker ] _default_handlers = [ PrefilterHandler, - AliasHandler, MacroHandler, MagicHandler, AutoHandler, diff --git a/IPython/core/profileapp.py b/IPython/core/profileapp.py index 9c6a529..4235789 100644 --- a/IPython/core/profileapp.py +++ b/IPython/core/profileapp.py @@ -11,7 +11,7 @@ Authors: """ #----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team +# Copyright (C) 2008 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. @@ -28,6 +28,7 @@ from IPython.core.application import ( BaseIPythonApplication, base_flags ) from IPython.core.profiledir import ProfileDir +from IPython.utils.importstring import import_item from IPython.utils.path import get_ipython_dir, get_ipython_package_dir from IPython.utils.traitlets import Unicode, Bool, Dict @@ -206,6 +207,8 @@ class ProfileCreate(BaseIPythonApplication): description = create_help examples = _create_examples auto_create = Bool(True, config=False) + def _log_format_default(self): + return "[%(name)s] %(message)s" def _copy_config_files_default(self): return True @@ -234,31 +237,32 @@ class ProfileCreate(BaseIPythonApplication): flags = Dict(create_flags) classes = [ProfileDir] + + def _import_app(self, app_path): + """import an app class""" + app = None + name = app_path.rsplit('.', 1)[-1] + try: + app = import_item(app_path) + except ImportError as e: + self.log.info("Couldn't import %s, config file will be excluded", name) + except Exception: + self.log.warn('Unexpected error importing %s', name, exc_info=True) + return app def init_config_files(self): super(ProfileCreate, self).init_config_files() # use local imports, since these classes may import from here from IPython.terminal.ipapp import TerminalIPythonApp apps = [TerminalIPythonApp] - try: - from IPython.qt.console.qtconsoleapp import IPythonQtConsoleApp - except Exception: - # this should be ImportError, but under weird circumstances - # this might be an AttributeError, or possibly others - # in any case, nothing should cause the profile creation to crash. - pass - else: - apps.append(IPythonQtConsoleApp) - try: - from IPython.html.notebookapp import NotebookApp - except ImportError: - pass - except Exception: - self.log.debug('Unexpected error when importing NotebookApp', - exc_info=True - ) - else: - apps.append(NotebookApp) + for app_path in ( + 'IPython.qt.console.qtconsoleapp.IPythonQtConsoleApp', + 'IPython.html.notebookapp.NotebookApp', + 'IPython.nbconvert.nbconvertapp.NbConvertApp', + ): + app = self._import_app(app_path) + if app is not None: + apps.append(app) if self.parallel: from IPython.parallel.apps.ipcontrollerapp import IPControllerApp from IPython.parallel.apps.ipengineapp import IPEngineApp diff --git a/IPython/core/pylabtools.py b/IPython/core/pylabtools.py index 81ee691..b6d9bf7 100644 --- a/IPython/core/pylabtools.py +++ b/IPython/core/pylabtools.py @@ -214,7 +214,11 @@ def find_gui_and_backend(gui=None, gui_select=None): # select backend based on requested gui backend = backends[gui] else: - backend = matplotlib.rcParams['backend'] + # We need to read the backend from the original data structure, *not* + # from mpl.rcParams, since a prior invocation of %matplotlib may have + # overwritten that. + # WARNING: this assumes matplotlib 1.1 or newer!! + backend = matplotlib.rcParamsOrig['backend'] # In this case, we need to find what the appropriate gui selection call # should be for IPython, so we can activate inputhook accordingly gui = backend2gui.get(backend, None) diff --git a/IPython/core/release.py b/IPython/core/release.py index e0f1f7e..8347967 100644 --- a/IPython/core/release.py +++ b/IPython/core/release.py @@ -26,7 +26,7 @@ _version_extra = 'dev' # _version_extra = 'rc1' # _version_extra = '' # Uncomment this for full releases -codename = 'An Afternoon Hack' +codename = 'Work in Progress' # Construct full version string from these. _ver = [_version_major, _version_minor, _version_patch] @@ -141,11 +141,8 @@ classifiers = [ 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', 'Topic :: System :: Distributed Computing', 'Topic :: System :: Shells' ] diff --git a/IPython/core/shellapp.py b/IPython/core/shellapp.py index 21785a2..032dd97 100644 --- a/IPython/core/shellapp.py +++ b/IPython/core/shellapp.py @@ -36,11 +36,14 @@ from IPython.utils.path import filefind from IPython.utils.traitlets import ( Unicode, Instance, List, Bool, CaselessStrEnum, Dict ) +from IPython.lib.inputhook import guis #----------------------------------------------------------------------------- # Aliases and Flags #----------------------------------------------------------------------------- +gui_keys = tuple(sorted([ key for key in guis if key is not None ])) + backend_keys = sorted(pylabtools.backends.keys()) backend_keys.insert(0, 'auto') @@ -175,8 +178,8 @@ class InteractiveShellApp(Configurable): module_to_run = Unicode('', config=True, help="Run the module as a script." ) - gui = CaselessStrEnum(('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx'), config=True, - help="Enable GUI event loop integration ('qt', 'wx', 'gtk', 'glut', 'pyglet', 'osx')." + gui = CaselessStrEnum(gui_keys, config=True, + help="Enable GUI event loop integration with any of {0}.".format(gui_keys) ) matplotlib = CaselessStrEnum(backend_keys, config=True, @@ -198,7 +201,7 @@ class InteractiveShellApp(Configurable): ) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') - user_ns = Dict(default_value=None) + user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new diff --git a/IPython/core/tests/test_alias.py b/IPython/core/tests/test_alias.py new file mode 100644 index 0000000..8ae57f6 --- /dev/null +++ b/IPython/core/tests/test_alias.py @@ -0,0 +1,41 @@ +from IPython.utils.capture import capture_output + +import nose.tools as nt + +def test_alias_lifecycle(): + name = 'test_alias1' + cmd = 'echo "Hello"' + am = _ip.alias_manager + am.clear_aliases() + am.define_alias(name, cmd) + assert am.is_alias(name) + nt.assert_equal(am.retrieve_alias(name), cmd) + nt.assert_in((name, cmd), am.aliases) + + # Test running the alias + orig_system = _ip.system + result = [] + _ip.system = result.append + try: + _ip.run_cell('%{}'.format(name)) + result = [c.strip() for c in result] + nt.assert_equal(result, [cmd]) + finally: + _ip.system = orig_system + + # Test removing the alias + am.undefine_alias(name) + assert not am.is_alias(name) + with nt.assert_raises(ValueError): + am.retrieve_alias(name) + nt.assert_not_in((name, cmd), am.aliases) + + +def test_alias_args_error(): + """Error expanding with wrong number of arguments""" + _ip.alias_manager.define_alias('parts', 'echo first %s second %s') + # capture stderr: + with capture_output() as cap: + _ip.run_cell('parts 1') + + nt.assert_equal(cap.stderr.split(':')[0], 'UsageError') \ No newline at end of file diff --git a/IPython/core/tests/test_debugger.py b/IPython/core/tests/test_debugger.py index f5f02e1..5d1b31f 100644 --- a/IPython/core/tests/test_debugger.py +++ b/IPython/core/tests/test_debugger.py @@ -117,8 +117,7 @@ def test_ipdb_magics(): ipdb> pinfo a Type: ExampleClass String Form:ExampleClass() - Namespace: Locals - File: ... + Namespace: Local... Docstring: Docstring for ExampleClass. Constructor Docstring:Docstring for ExampleClass.__init__ ipdb> continue diff --git a/IPython/core/tests/test_fakemodule.py b/IPython/core/tests/test_fakemodule.py deleted file mode 100644 index d12bd5d..0000000 --- a/IPython/core/tests/test_fakemodule.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for the FakeModule objects. -""" - -import nose.tools as nt - -from IPython.core.fakemodule import FakeModule - -# Make a fakemod and check a few properties -def test_mk_fakemod(): - fm = FakeModule() - yield nt.assert_true,fm - yield nt.assert_true,lambda : hasattr(fm,'__file__') - -def test_mk_fakemod_fromdict(): - """Test making a FakeModule object with initial data""" - fm = FakeModule(dict(hello=True)) - nt.assert_true(fm.hello) diff --git a/IPython/core/tests/test_handlers.py b/IPython/core/tests/test_handlers.py index 5601717..20f956a 100644 --- a/IPython/core/tests/test_handlers.py +++ b/IPython/core/tests/test_handlers.py @@ -45,28 +45,6 @@ def run(tests): def test_handlers(): - # alias expansion - - # We're using 'true' as our syscall of choice because it doesn't - # write anything to stdout. - - # Turn off actual execution of aliases, because it's noisy - old_system_cmd = ip.system - ip.system = lambda cmd: None - - - ip.alias_manager.alias_table['an_alias'] = (0, 'true') - # These are useful for checking a particular recursive alias issue - ip.alias_manager.alias_table['top'] = (0, 'd:/cygwin/top') - ip.alias_manager.alias_table['d'] = (0, 'true') - run([(i,py3compat.u_format(o)) for i,o in \ - [("an_alias", "get_ipython().system({u}'true ')"), # alias - # Below: recursive aliases should expand whitespace-surrounded - # chars, *not* initial chars which happen to be aliases: - ("top", "get_ipython().system({u}'d:/cygwin/top ')"), - ]]) - ip.system = old_system_cmd - call_idx = CallableIndexable() ip.user_ns['call_idx'] = call_idx diff --git a/IPython/core/tests/test_imports.py b/IPython/core/tests/test_imports.py index 04d83f4..88caef0 100644 --- a/IPython/core/tests/test_imports.py +++ b/IPython/core/tests/test_imports.py @@ -9,9 +9,6 @@ def test_import_crashhandler(): def test_import_debugger(): from IPython.core import debugger -def test_import_fakemodule(): - from IPython.core import fakemodule - def test_import_excolors(): from IPython.core import excolors diff --git a/IPython/core/tests/test_inputtransformer.py b/IPython/core/tests/test_inputtransformer.py index 56d4c55..f9fe368 100644 --- a/IPython/core/tests/test_inputtransformer.py +++ b/IPython/core/tests/test_inputtransformer.py @@ -183,9 +183,18 @@ syntax_ml = \ ('... 456"""','456"""'), ], [('a="""','a="""'), + ('>>> 123','123'), + ('... 456"""','456"""'), + ], + [('a="""','a="""'), ('123','123'), ('... 456"""','... 456"""'), ], + [('....__class__','....__class__'), + ], + [('a=5', 'a=5'), + ('...', ''), + ], [('>>> def f(x):', 'def f(x):'), ('...', ''), ('... return x', ' return x'), @@ -205,6 +214,10 @@ syntax_ml = \ (' ...: 456"""','456"""'), ], [('a="""','a="""'), + ('In [1]: 123','123'), + (' ...: 456"""','456"""'), + ], + [('a="""','a="""'), ('123','123'), (' ...: 456"""',' ...: 456"""'), ], @@ -237,6 +250,12 @@ syntax_ml = \ ], ], + multiline_string = + [ [("'''foo?", None), + ("bar'''", "'''foo?\nbar'''"), + ], + ], + leading_indent = [ [(' print "hi"','print "hi"'), ], diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index 954f60c..ee6569c 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -106,17 +106,6 @@ class InteractiveShellTestCase(unittest.TestCase): ip.run_cell('a = """\n%exit\n"""') self.assertEqual(ip.user_ns['a'], '\n%exit\n') - def test_alias_crash(self): - """Errors in prefilter can't crash IPython""" - ip.run_cell('%alias parts echo first %s second %s') - # capture stderr: - save_err = io.stderr - io.stderr = StringIO() - ip.run_cell('parts 1') - err = io.stderr.getvalue() - io.stderr = save_err - self.assertEqual(err.split(':')[0], 'ERROR') - def test_trailing_newline(self): """test that running !(command) does not raise a SyntaxError""" ip.run_cell('!(true)\n', False) diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index 7740524..d0953dd 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -48,16 +48,16 @@ class DummyMagics(magic.Magics): pass def test_rehashx(): # clear up everything _ip = get_ipython() - _ip.alias_manager.alias_table.clear() + _ip.alias_manager.clear_aliases() del _ip.db['syscmdlist'] _ip.magic('rehashx') # Practically ALL ipython development systems will have more than 10 aliases - nt.assert_true(len(_ip.alias_manager.alias_table) > 10) - for key, val in _ip.alias_manager.alias_table.iteritems(): + nt.assert_true(len(_ip.alias_manager.aliases) > 10) + for name, cmd in _ip.alias_manager.aliases: # we must strip dots from alias names - nt.assert_not_in('.', key) + nt.assert_not_in('.', name) # rehashx must fill up syscmdlist scoms = _ip.db['syscmdlist'] @@ -487,7 +487,21 @@ def test_timeit_special_syntax(): # cell mode test _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2') nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2') - + +def test_timeit_return(): + """ + test wether timeit -o return object + """ + + res = _ip.run_line_magic('timeit','-n10 -r10 -o 1') + assert(res is not None) + +def test_timeit_quiet(): + """ + test quiet option of timeit magic + """ + with tt.AssertNotPrints("loops"): + _ip.run_cell("%timeit -n1 -r1 -q 1") @dec.skipif(execution.profile is None) def test_prun_special_syntax(): diff --git a/IPython/core/tests/test_magic_arguments.py b/IPython/core/tests/test_magic_arguments.py index 5d667d6..92ae04a 100644 --- a/IPython/core/tests/test_magic_arguments.py +++ b/IPython/core/tests/test_magic_arguments.py @@ -6,12 +6,11 @@ # The full license is in the file COPYING.txt, distributed with this software. #----------------------------------------------------------------------------- -from nose.tools import assert_equal, assert_true +import argparse +from nose.tools import assert_equal -from IPython.external import argparse from IPython.core.magic_arguments import (argument, argument_group, kwds, magic_arguments, parse_argstring, real_name) -from IPython.testing.decorators import parametric @magic_arguments() @@ -74,48 +73,46 @@ def foo(self, args): return parse_argstring(foo, args) -@parametric def test_magic_arguments(): - # Ideally, these would be doctests, but I could not get it to work. - yield assert_equal(magic_foo1.__doc__, '%foo1 [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') - yield assert_equal(getattr(magic_foo1, 'argcmd_name', None), None) - yield assert_equal(real_name(magic_foo1), 'foo1') - yield assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None)) - yield assert_true(hasattr(magic_foo1, 'has_arguments')) - - yield assert_equal(magic_foo2.__doc__, '%foo2\n\n A docstring.\n') - yield assert_equal(getattr(magic_foo2, 'argcmd_name', None), None) - yield assert_equal(real_name(magic_foo2), 'foo2') - yield assert_equal(magic_foo2(None, ''), argparse.Namespace()) - yield assert_true(hasattr(magic_foo2, 'has_arguments')) - - yield assert_equal(magic_foo3.__doc__, '%foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n\nGroup:\n -b BAR, --bar BAR a grouped argument\n\nSecond Group:\n -z BAZ, --baz BAZ another grouped argument\n') - yield assert_equal(getattr(magic_foo3, 'argcmd_name', None), None) - yield assert_equal(real_name(magic_foo3), 'foo3') - yield assert_equal(magic_foo3(None, ''), + assert_equal(magic_foo1.__doc__, '%foo1 [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(getattr(magic_foo1, 'argcmd_name', None), None) + assert_equal(real_name(magic_foo1), 'foo1') + assert_equal(magic_foo1(None, ''), argparse.Namespace(foo=None)) + assert hasattr(magic_foo1, 'has_arguments') + + assert_equal(magic_foo2.__doc__, '%foo2\n\n A docstring.\n') + assert_equal(getattr(magic_foo2, 'argcmd_name', None), None) + assert_equal(real_name(magic_foo2), 'foo2') + assert_equal(magic_foo2(None, ''), argparse.Namespace()) + assert hasattr(magic_foo2, 'has_arguments') + + assert_equal(magic_foo3.__doc__, '%foo3 [-f FOO] [-b BAR] [-z BAZ]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n\nGroup:\n -b BAR, --bar BAR a grouped argument\n\nSecond Group:\n -z BAZ, --baz BAZ another grouped argument\n') + assert_equal(getattr(magic_foo3, 'argcmd_name', None), None) + assert_equal(real_name(magic_foo3), 'foo3') + assert_equal(magic_foo3(None, ''), argparse.Namespace(bar=None, baz=None, foo=None)) - yield assert_true(hasattr(magic_foo3, 'has_arguments')) - - yield assert_equal(magic_foo4.__doc__, '%foo4 [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') - yield assert_equal(getattr(magic_foo4, 'argcmd_name', None), None) - yield assert_equal(real_name(magic_foo4), 'foo4') - yield assert_equal(magic_foo4(None, ''), argparse.Namespace()) - yield assert_true(hasattr(magic_foo4, 'has_arguments')) - - yield assert_equal(magic_foo5.__doc__, '%frobnicate [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') - yield assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate') - yield assert_equal(real_name(magic_foo5), 'frobnicate') - yield assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None)) - yield assert_true(hasattr(magic_foo5, 'has_arguments')) - - yield assert_equal(magic_magic_foo.__doc__, '%magic_foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') - yield assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None) - yield assert_equal(real_name(magic_magic_foo), 'magic_foo') - yield assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None)) - yield assert_true(hasattr(magic_magic_foo, 'has_arguments')) - - yield assert_equal(foo.__doc__, '%foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') - yield assert_equal(getattr(foo, 'argcmd_name', None), None) - yield assert_equal(real_name(foo), 'foo') - yield assert_equal(foo(None, ''), argparse.Namespace(foo=None)) - yield assert_true(hasattr(foo, 'has_arguments')) + assert hasattr(magic_foo3, 'has_arguments') + + assert_equal(magic_foo4.__doc__, '%foo4 [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(getattr(magic_foo4, 'argcmd_name', None), None) + assert_equal(real_name(magic_foo4), 'foo4') + assert_equal(magic_foo4(None, ''), argparse.Namespace()) + assert hasattr(magic_foo4, 'has_arguments') + + assert_equal(magic_foo5.__doc__, '%frobnicate [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(getattr(magic_foo5, 'argcmd_name', None), 'frobnicate') + assert_equal(real_name(magic_foo5), 'frobnicate') + assert_equal(magic_foo5(None, ''), argparse.Namespace(foo=None)) + assert hasattr(magic_foo5, 'has_arguments') + + assert_equal(magic_magic_foo.__doc__, '%magic_foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(getattr(magic_magic_foo, 'argcmd_name', None), None) + assert_equal(real_name(magic_magic_foo), 'magic_foo') + assert_equal(magic_magic_foo(None, ''), argparse.Namespace(foo=None)) + assert hasattr(magic_magic_foo, 'has_arguments') + + assert_equal(foo.__doc__, '%foo [-f FOO]\n\n A docstring.\n\noptional arguments:\n -f FOO, --foo FOO an argument\n') + assert_equal(getattr(foo, 'argcmd_name', None), None) + assert_equal(real_name(foo), 'foo') + assert_equal(foo(None, ''), argparse.Namespace(foo=None)) + assert hasattr(foo, 'has_arguments') diff --git a/IPython/core/tests/test_prefilter.py b/IPython/core/tests/test_prefilter.py index 6f0b326..b54abf0 100644 --- a/IPython/core/tests/test_prefilter.py +++ b/IPython/core/tests/test_prefilter.py @@ -6,7 +6,6 @@ import nose.tools as nt from IPython.core.prefilter import AutocallChecker -from IPython.testing import decorators as dec from IPython.testing.globalipapp import get_ipython #----------------------------------------------------------------------------- @@ -14,7 +13,6 @@ from IPython.testing.globalipapp import get_ipython #----------------------------------------------------------------------------- ip = get_ipython() -@dec.parametric def test_prefilter(): """Test user input conversions""" @@ -23,19 +21,18 @@ def test_prefilter(): ] for raw, correct in pairs: - yield nt.assert_equal(ip.prefilter(raw), correct) + nt.assert_equal(ip.prefilter(raw), correct) -@dec.parametric def test_autocall_binops(): """See https://github.com/ipython/ipython/issues/81""" ip.magic('autocall 2') f = lambda x: x ip.user_ns['f'] = f try: - yield nt.assert_equal(ip.prefilter('f 1'),'f(1)') + nt.assert_equal(ip.prefilter('f 1'),'f(1)') for t in ['f +1', 'f -1']: - yield nt.assert_equal(ip.prefilter(t), t) + nt.assert_equal(ip.prefilter(t), t) # Run tests again with a more permissive exclude_regexp, which will # allow transformation of binary operations ('f -1' -> 'f(-1)'). @@ -47,8 +44,8 @@ def test_autocall_binops(): ac.exclude_regexp = r'^[,&^\|\*/]|^is |^not |^in |^and |^or ' pm.sort_checkers() - yield nt.assert_equal(ip.prefilter('f -1'), 'f(-1)') - yield nt.assert_equal(ip.prefilter('f +1'), 'f(+1)') + nt.assert_equal(ip.prefilter('f -1'), 'f(-1)') + nt.assert_equal(ip.prefilter('f +1'), 'f(+1)') finally: pm.unregister_checker(ac) finally: @@ -56,7 +53,6 @@ def test_autocall_binops(): del ip.user_ns['f'] -@dec.parametric def test_issue_114(): """Check that multiline string literals don't expand as magic see http://github.com/ipython/ipython/issues/114""" @@ -70,7 +66,7 @@ def test_issue_114(): try: for mgk in ip.magics_manager.lsmagic()['line']: raw = template % mgk - yield nt.assert_equal(ip.prefilter(raw), raw) + nt.assert_equal(ip.prefilter(raw), raw) finally: ip.prefilter_manager.multi_line_specials = msp diff --git a/IPython/core/tests/test_pylabtools.py b/IPython/core/tests/test_pylabtools.py index 4ee672a..16ced0f 100644 --- a/IPython/core/tests/test_pylabtools.py +++ b/IPython/core/tests/test_pylabtools.py @@ -24,7 +24,6 @@ import numpy as np # Our own imports from IPython.core.interactiveshell import InteractiveShell -from IPython.testing import decorators as dec from .. import pylabtools as pt #----------------------------------------------------------------------------- @@ -39,11 +38,10 @@ from .. import pylabtools as pt # Classes and functions #----------------------------------------------------------------------------- -@dec.parametric def test_figure_to_svg(): # simple empty-figure test fig = plt.figure() - yield nt.assert_equal(pt.print_figure(fig, 'svg'), None) + nt.assert_equal(pt.print_figure(fig, 'svg'), None) plt.close('all') @@ -53,7 +51,7 @@ def test_figure_to_svg(): ax.plot([1,2,3]) plt.draw() svg = pt.print_figure(fig, 'svg')[:100].lower() - yield nt.assert_true('doctype svg' in svg) + nt.assert_in(b'doctype svg', svg) def test_import_pylab(): diff --git a/IPython/core/tests/test_run.py b/IPython/core/tests/test_run.py index 9ec10f2..9183d96 100644 --- a/IPython/core/tests/test_run.py +++ b/IPython/core/tests/test_run.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import functools import os +from os.path import join as pjoin import random import sys import tempfile @@ -359,6 +360,17 @@ tclass.py: deleting object: C-third self.mktmp(src) _ip.magic('run -t -N 1 %s' % self.fname) _ip.magic('run -t -N 10 %s' % self.fname) + + def test_ignore_sys_exit(self): + """Test the -e option to ignore sys.exit()""" + src = "import sys; sys.exit(1)" + self.mktmp(src) + with tt.AssertPrints('SystemExit'): + _ip.magic('run %s' % self.fname) + + with tt.AssertNotPrints('SystemExit'): + _ip.magic('run -e %s' % self.fname) + class TestMagicRunWithPackage(unittest.TestCase): @@ -398,6 +410,7 @@ class TestMagicRunWithPackage(unittest.TestCase): self.tempdir.cleanup() def check_run_submodule(self, submodule, opts=''): + _ip.user_ns.pop('x', None) _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts)) self.assertEqual(_ip.user_ns['x'], self.value, 'Variable `x` is not loaded from module `{0}`.' @@ -430,3 +443,16 @@ class TestMagicRunWithPackage(unittest.TestCase): @with_fake_debugger def test_debug_run_submodule_with_relative_import(self): self.check_run_submodule('relative', '-d') + +def test_run__name__(): + with TemporaryDirectory() as td: + path = pjoin(td, 'foo.py') + with open(path, 'w') as f: + f.write("q = __name__") + + _ip.user_ns.pop('q', None) + _ip.magic('run {}'.format(path)) + nt.assert_equal(_ip.user_ns.pop('q'), '__main__') + + _ip.magic('run -n {}'.format(path)) + nt.assert_equal(_ip.user_ns.pop('q'), 'foo') diff --git a/IPython/core/tests/test_ultratb.py b/IPython/core/tests/test_ultratb.py index c1530d2..d4f4369 100644 --- a/IPython/core/tests/test_ultratb.py +++ b/IPython/core/tests/test_ultratb.py @@ -108,8 +108,33 @@ class IndentationErrorTest(unittest.TestCase): with tt.AssertPrints("zoon()", suppress=False): ip.magic('run %s' % fname) +se_file_1 = """1 +2 +7/ +""" + +se_file_2 = """7/ +""" + class SyntaxErrorTest(unittest.TestCase): def test_syntaxerror_without_lineno(self): with tt.AssertNotPrints("TypeError"): with tt.AssertPrints("line unknown"): ip.run_cell("raise SyntaxError()") + + def test_changing_py_file(self): + with TemporaryDirectory() as td: + fname = os.path.join(td, "foo.py") + with open(fname, 'w') as f: + f.write(se_file_1) + + with tt.AssertPrints(["7/", "SyntaxError"]): + ip.magic("run " + fname) + + # Modify the file + with open(fname, 'w') as f: + f.write(se_file_2) + + # The SyntaxError should point to the correct line + with tt.AssertPrints(["7/", "SyntaxError"]): + ip.magic("run " + fname) diff --git a/IPython/core/ultratb.py b/IPython/core/ultratb.py index 4261339..ebab505 100644 --- a/IPython/core/ultratb.py +++ b/IPython/core/ultratb.py @@ -8,7 +8,8 @@ ColorTB class is a solution to that problem. It colors the different parts of a traceback in a manner similar to what you would expect from a syntax-highlighting text editor. -Installation instructions for ColorTB: +Installation instructions for ColorTB:: + import sys,ultratb sys.excepthook = ultratb.ColorTB() @@ -21,7 +22,7 @@ but kind of neat, and maybe useful for long-running programs that you believe are bug-free. If a crash *does* occur in that type of program you want details. Give it a shot--you'll love it or you'll hate it. -Note: +.. note:: The Verbose mode prints the variables currently visible where the exception happened (shortening their strings if too long). This can potentially be @@ -36,25 +37,28 @@ Note: Verbose). -Installation instructions for ColorTB: +Installation instructions for ColorTB:: + import sys,ultratb sys.excepthook = ultratb.VerboseTB() Note: Much of the code in this module was lifted verbatim from the standard library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'. -* Color schemes +Color schemes +------------- + The colors are defined in the class TBTools through the use of the ColorSchemeTable class. Currently the following exist: - NoColor: allows all of this module to be used in any terminal (the color - escapes are just dummy blank strings). + escapes are just dummy blank strings). - Linux: is meant to look good in a terminal like the Linux console (black - or very dark background). + or very dark background). - LightBG: similar to Linux but swaps dark/light colors to be more readable - in light background terminals. + in light background terminals. You can implement other color schemes easily, the syntax is fairly self-explanatory. Please send back new schemes you develop to the author for @@ -359,8 +363,8 @@ class TBTools(object): Valid values are: - None: the default, which means that IPython will dynamically resolve - to io.stdout. This ensures compatibility with most tools, including - Windows (where plain stdout doesn't recognize ANSI escapes). + to io.stdout. This ensures compatibility with most tools, including + Windows (where plain stdout doesn't recognize ANSI escapes). - Any object with 'write' and 'flush' attributes. """ @@ -974,9 +978,9 @@ class VerboseTB(TBTools): Keywords: - force(False): by default, this routine checks the instance call_pdb - flag and does not actually invoke the debugger if the flag is false. - The 'force' option forces the debugger to activate even if the flag - is false. + flag and does not actually invoke the debugger if the flag is false. + The 'force' option forces the debugger to activate even if the flag + is false. If the call_pdb flag is set, the pdb interactive debugger is invoked. In all cases, the self.tb reference to the current traceback @@ -1193,6 +1197,20 @@ class SyntaxTB(ListTB): self.last_syntax_error = value ListTB.__call__(self,etype,value,elist) + def structured_traceback(self, etype, value, elist, tb_offset=None, + context=5): + # If the source file has been edited, the line in the syntax error can + # be wrong (retrieved from an outdated cache). This replaces it with + # the current value. + if isinstance(value.filename, py3compat.string_types) \ + and isinstance(value.lineno, int): + linecache.checkcache(value.filename) + newtext = ulinecache.getline(value.filename, value.lineno) + if newtext: + value.text = newtext + return super(SyntaxTB, self).structured_traceback(etype, value, elist, + tb_offset=tb_offset, context=context) + def clear_err_state(self): """Return the current error state and clear it""" e = self.last_syntax_error diff --git a/IPython/extensions/cythonmagic.py b/IPython/extensions/cythonmagic.py index 4c1d121..2908941 100644 --- a/IPython/extensions/cythonmagic.py +++ b/IPython/extensions/cythonmagic.py @@ -4,9 +4,18 @@ Cython related magics ===================== +Magic command interface for interactive work with Cython + +.. note:: + + The ``Cython`` package needs to be installed separately. It + can be obtained using ``easy_install`` or ``pip``. + Usage ===== +To enable the magics below, execute ``%load_ext cythonmagic``. + ``%%cython`` {CYTHON_DOC} diff --git a/IPython/extensions/octavemagic.py b/IPython/extensions/octavemagic.py index 5597099..1725a34 100644 --- a/IPython/extensions/octavemagic.py +++ b/IPython/extensions/octavemagic.py @@ -11,9 +11,13 @@ Magics for interacting with Octave via oct2py. The ``oct2py`` module needs to be installed separately and can be obtained using ``easy_install`` or ``pip``. + You will also need a working copy of GNU Octave. + Usage ===== +To enable the magics below, execute ``%load_ext octavemagic``. + ``%octave`` {OCTAVE_DOC} diff --git a/IPython/extensions/rmagic.py b/IPython/extensions/rmagic.py index 453d78c..4854245 100644 --- a/IPython/extensions/rmagic.py +++ b/IPython/extensions/rmagic.py @@ -6,9 +6,18 @@ Rmagic Magic command interface for interactive work with R via rpy2 +.. note:: + + The ``rpy2`` package needs to be installed separately. It + can be obtained using ``easy_install`` or ``pip``. + + You will also need a working copy of R. + Usage ===== +To enable the magics below, execute ``%load_ext rmagic``. + ``%R`` {R_DOC} @@ -425,12 +434,12 @@ class RMagics(Magics): is printed if it would printed be when evaluating the same code within a standard R REPL. - Nothing is returned to python by default in cell mode. + Nothing is returned to python by default in cell mode:: In [10]: %%R ....: Y = c(2,4,3,9) ....: summary(lm(Y~X)) - + Call: lm(formula = Y ~ X) @@ -447,9 +456,9 @@ class RMagics(Magics): Multiple R-squared: 0.6993,Adjusted R-squared: 0.549 F-statistic: 4.651 on 1 and 2 DF, p-value: 0.1638 - In the notebook, plots are published as the output of the cell. + In the notebook, plots are published as the output of the cell:: - %R plot(X, Y) + %R plot(X, Y) will create a scatter plot of X bs Y. @@ -475,19 +484,19 @@ class RMagics(Magics): * If the cell is not None, the magic returns None. * If the cell evaluates as False, the resulting value is returned - unless the final line prints something to the console, in - which case None is returned. + unless the final line prints something to the console, in + which case None is returned. * If the final line results in a NULL value when evaluated - by rpy2, then None is returned. + by rpy2, then None is returned. * No attempt is made to convert the final value to a structured array. - Use the --dataframe flag or %Rget to push / return a structured array. + Use the --dataframe flag or %Rget to push / return a structured array. * If the -n flag is present, there is no return value. * A trailing ';' will also result in no return value as the last - value in the line is an empty string. + value in the line is an empty string. The --dataframe argument will attempt to return structured arrays. This is useful for dataframes with diff --git a/IPython/extensions/storemagic.py b/IPython/extensions/storemagic.py index 8759b2a..aa7a7c1 100644 --- a/IPython/extensions/storemagic.py +++ b/IPython/extensions/storemagic.py @@ -25,10 +25,11 @@ To automatically restore stored variables at startup, add this to your import inspect, os, sys, textwrap # Our own +from IPython.config.configurable import Configurable from IPython.core.error import UsageError -from IPython.core.fakemodule import FakeModule from IPython.core.magic import Magics, magics_class, line_magic from IPython.testing.skipdoctest import skip_doctest +from IPython.utils.traitlets import Bool #----------------------------------------------------------------------------- # Functions and classes @@ -68,10 +69,23 @@ def restore_data(ip): @magics_class -class StoreMagics(Magics): +class StoreMagics(Magics, Configurable): """Lightweight persistence for python variables. Provides the %store magic.""" + + autorestore = Bool(False, config=True, help= + """If True, any %store-d variables will be automatically restored + when IPython starts. + """ + ) + + def __init__(self, shell): + Configurable.__init__(self, config=shell.config) + Magics.__init__(self, shell=shell) + self.shell.configurables.append(self) + if self.autorestore: + restore_data(self.shell) @skip_doctest @line_magic @@ -196,20 +210,21 @@ class StoreMagics(Magics): obj = ip.user_ns[args[0]] except KeyError: # it might be an alias - # This needs to be refactored to use the new AliasManager stuff. - if args[0] in ip.alias_manager: - name = args[0] - nargs, cmd = ip.alias_manager.alias_table[ name ] - staliases = db.get('stored_aliases',{}) - staliases[ name ] = cmd - db['stored_aliases'] = staliases - print "Alias stored: %s (%s)" % (name, cmd) - return - else: - raise UsageError("Unknown variable '%s'" % args[0]) + name = args[0] + try: + cmd = ip.alias_manager.retrieve_alias(name) + except ValueError: + raise UsageError("Unknown variable '%s'" % name) + + staliases = db.get('stored_aliases',{}) + staliases[name] = cmd + db['stored_aliases'] = staliases + print "Alias stored: %s (%s)" % (name, cmd) + return else: - if isinstance(inspect.getmodule(obj), FakeModule): + modname = getattr(inspect.getmodule(obj), '__name__', '') + if modname == '__main__': print textwrap.dedent("""\ Warning:%s is %s Proper storage of interactively declared classes (or instances @@ -225,3 +240,4 @@ class StoreMagics(Magics): def load_ipython_extension(ip): """Load the extension in IPython.""" ip.register_magics(StoreMagics) + diff --git a/IPython/extensions/tests/test_storemagic.py b/IPython/extensions/tests/test_storemagic.py index ac6fb97..a5a15ba 100644 --- a/IPython/extensions/tests/test_storemagic.py +++ b/IPython/extensions/tests/test_storemagic.py @@ -1,5 +1,6 @@ import tempfile, os +from IPython.config.loader import Config import nose.tools as nt ip = get_ipython() @@ -26,7 +27,24 @@ def test_store_restore(): # Check restoring ip.magic('store -r') nt.assert_equal(ip.user_ns['foo'], 78) - nt.assert_in('bar', ip.alias_manager.alias_table) + assert ip.alias_manager.is_alias('bar') nt.assert_in(os.path.realpath(tmpd), ip.user_ns['_dh']) os.rmdir(tmpd) + +def test_autorestore(): + ip.user_ns['foo'] = 95 + ip.magic('store foo') + del ip.user_ns['foo'] + c = Config() + c.StoreMagics.autorestore = False + orig_config = ip.config + try: + ip.config = c + ip.extension_manager.reload_extension('storemagic') + nt.assert_not_in('foo', ip.user_ns) + c.StoreMagics.autorestore = True + ip.extension_manager.reload_extension('storemagic') + nt.assert_equal(ip.user_ns['foo'], 95) + finally: + ip.config = orig_config diff --git a/IPython/external/argparse/__init__.py b/IPython/external/argparse/__init__.py deleted file mode 100644 index 699f017..0000000 --- a/IPython/external/argparse/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from IPython.utils.version import check_version -try: - import argparse - # don't use system argparse if older than 1.1: - if not check_version(argparse.__version__, '1.1'): - raise ImportError - else: - from argparse import * - from argparse import SUPPRESS -except (ImportError, AttributeError): - from _argparse import * - from _argparse import SUPPRESS diff --git a/IPython/external/argparse/_argparse.py b/IPython/external/argparse/_argparse.py deleted file mode 100644 index a060129..0000000 --- a/IPython/external/argparse/_argparse.py +++ /dev/null @@ -1,2353 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright © 2006-2009 Steven J. Bethard . -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy -# of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Command-line parsing library - -This module is an optparse-inspired command-line parsing library that: - - - handles both optional and positional arguments - - produces highly informative usage messages - - supports parsers that dispatch to sub-parsers - -The following is a simple usage example that sums integers from the -command-line and writes the result to a file:: - - parser = argparse.ArgumentParser( - description='sum the integers at the command line') - parser.add_argument( - 'integers', metavar='int', nargs='+', type=int, - help='an integer to be summed') - parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), - help='the file where the sum should be written') - args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() - -The module contains the following public classes: - - - ArgumentParser -- The main entry point for command-line parsing. As the - example above shows, the add_argument() method is used to populate - the parser with actions for optional and positional arguments. Then - the parse_args() method is invoked to convert the args at the - command-line into an object with attributes. - - - ArgumentError -- The exception raised by ArgumentParser objects when - there are errors with the parser's actions. Errors raised while - parsing the command-line are caught by ArgumentParser and emitted - as command-line messages. - - - FileType -- A factory for defining types of files to be created. As the - example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. - - - Action -- The base class for parser actions. Typically actions are - selected by passing strings like 'store_true' or 'append_const' to - the action= argument of add_argument(). However, for greater - customization of ArgumentParser actions, subclasses of Action may - be defined and passed as the action= argument. - - - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, - ArgumentDefaultsHelpFormatter -- Formatter classes which - may be passed as the formatter_class= argument to the - ArgumentParser constructor. HelpFormatter is the default, - RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser - not to change the formatting for help text, and - ArgumentDefaultsHelpFormatter adds information about argument defaults - to the help. - -All other classes in this module are considered implementation details. -(Also note that HelpFormatter and RawDescriptionHelpFormatter are only -considered public as object names -- the API of the formatter objects is -still considered an implementation detail.) -""" - -__version__ = '1.1' -__all__ = [ - 'ArgumentParser', - 'ArgumentError', - 'Namespace', - 'Action', - 'FileType', - 'HelpFormatter', - 'RawDescriptionHelpFormatter', - 'RawTextHelpFormatter', - 'ArgumentDefaultsHelpFormatter', -] - - -import copy as _copy -import os as _os -import re as _re -import sys as _sys -import textwrap as _textwrap - -from gettext import gettext as _ - -try: - _set = set -except NameError: - from sets import Set as _set - -try: - _basestring = basestring -except NameError: - _basestring = str - -try: - _sorted = sorted -except NameError: - - def _sorted(iterable, reverse=False): - result = list(iterable) - result.sort() - if reverse: - result.reverse() - return result - - -def _callable(obj): - return hasattr(obj, '__call__') or hasattr(obj, '__bases__') - -# silence Python 2.6 buggy warnings about Exception.message -if _sys.version_info[:2] == (2, 6): - import warnings - warnings.filterwarnings( - action='ignore', - message='BaseException.message has been deprecated as of Python 2.6', - category=DeprecationWarning, - module='argparse') - - -SUPPRESS = '==SUPPRESS==' - -OPTIONAL = '?' -ZERO_OR_MORE = '*' -ONE_OR_MORE = '+' -PARSER = 'A...' -REMAINDER = '...' - -# ============================= -# Utility functions and classes -# ============================= - -class _AttributeHolder(object): - """Abstract base class that provides __repr__. - - The __repr__ method returns a string in the format:: - ClassName(attr=name, attr=name, ...) - The attributes are determined either by a class-level attribute, - '_kwarg_names', or by inspecting the instance __dict__. - """ - - def __repr__(self): - type_name = type(self).__name__ - arg_strings = [] - for arg in self._get_args(): - arg_strings.append(repr(arg)) - for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) - return '%s(%s)' % (type_name, ', '.join(arg_strings)) - - def _get_kwargs(self): - return _sorted(self.__dict__.items()) - - def _get_args(self): - return [] - - -def _ensure_value(namespace, name, value): - if getattr(namespace, name, None) is None: - setattr(namespace, name, value) - return getattr(namespace, name) - - -# =============== -# Formatting Help -# =============== - -class HelpFormatter(object): - """Formatter for generating usage messages and argument help strings. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - - # default setting for width - if width is None: - try: - width = int(_os.environ['COLUMNS']) - except (KeyError, ValueError): - width = 80 - width -= 2 - - self._prog = prog - self._indent_increment = indent_increment - self._max_help_position = max_help_position - self._width = width - - self._current_indent = 0 - self._level = 0 - self._action_max_length = 0 - - self._root_section = self._Section(self, None) - self._current_section = self._root_section - - self._whitespace_matcher = _re.compile(r'\s+') - self._long_break_matcher = _re.compile(r'\n\n\n+') - - # =============================== - # Section and indentation methods - # =============================== - def _indent(self): - self._current_indent += self._indent_increment - self._level += 1 - - def _dedent(self): - self._current_indent -= self._indent_increment - assert self._current_indent >= 0, 'Indent decreased below 0.' - self._level -= 1 - - class _Section(object): - - def __init__(self, formatter, parent, heading=None): - self.formatter = formatter - self.parent = parent - self.heading = heading - self.items = [] - - def format_help(self): - # format the indented section - if self.parent is not None: - self.formatter._indent() - join = self.formatter._join_parts - for func, args in self.items: - func(*args) - item_help = join([func(*args) for func, args in self.items]) - if self.parent is not None: - self.formatter._dedent() - - # return nothing if the section was empty - if not item_help: - return '' - - # add the heading if the section was non-empty - if self.heading is not SUPPRESS and self.heading is not None: - current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) - else: - heading = '' - - # join the section-initial newline, the heading and the help - return join(['\n', heading, item_help, '\n']) - - def _add_item(self, func, args): - self._current_section.items.append((func, args)) - - # ======================== - # Message building methods - # ======================== - def start_section(self, heading): - self._indent() - section = self._Section(self, self._current_section, heading) - self._add_item(section.format_help, []) - self._current_section = section - - def end_section(self): - self._current_section = self._current_section.parent - self._dedent() - - def add_text(self, text): - if text is not SUPPRESS and text is not None: - self._add_item(self._format_text, [text]) - - def add_usage(self, usage, actions, groups, prefix=None): - if usage is not SUPPRESS: - args = usage, actions, groups, prefix - self._add_item(self._format_usage, args) - - def add_argument(self, action): - if action.help is not SUPPRESS: - - # find all invocations - get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] - for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) - - # update the maximum item length - invocation_length = max([len(s) for s in invocations]) - action_length = invocation_length + self._current_indent - self._action_max_length = max(self._action_max_length, - action_length) - - # add the item to the list - self._add_item(self._format_action, [action]) - - def add_arguments(self, actions): - for action in actions: - self.add_argument(action) - - # ======================= - # Help-formatting methods - # ======================= - def format_help(self): - help = self._root_section.format_help() - if help: - help = self._long_break_matcher.sub('\n\n', help) - help = help.strip('\n') + '\n' - return help - - def _join_parts(self, part_strings): - return ''.join([part - for part in part_strings - if part and part is not SUPPRESS]) - - def _format_usage(self, usage, actions, groups, prefix): - if prefix is None: - prefix = _('usage: ') - - # if usage is specified, use that - if usage is not None: - usage = usage % dict(prog=self._prog) - - # if no optionals or positionals are available, usage is just prog - elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) - - # if optionals and positionals are available, calculate usage - elif usage is None: - prog = '%(prog)s' % dict(prog=self._prog) - - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - - # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) - - # wrap the usage parts if it's too long - text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: - - # break usage into wrappable parts - part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage - - # helper for wrapping lines - def get_lines(parts, indent, prefix=None): - lines = [] - line = [] - if prefix is not None: - line_len = len(prefix) - 1 - else: - line_len = len(indent) - 1 - for part in parts: - if line_len + 1 + len(part) > text_width: - lines.append(indent + ' '.join(line)) - line = [] - line_len = len(indent) - 1 - line.append(part) - line_len += len(part) + 1 - if line: - lines.append(indent + ' '.join(line)) - if prefix is not None: - lines[0] = lines[0][len(indent):] - return lines - - # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) - if opt_parts: - lines = get_lines([prog] + opt_parts, indent, prefix) - lines.extend(get_lines(pos_parts, indent)) - elif pos_parts: - lines = get_lines([prog] + pos_parts, indent, prefix) - else: - lines = [prog] - - # if prog is long, put it on its own line - else: - indent = ' ' * len(prefix) - parts = opt_parts + pos_parts - lines = get_lines(parts, indent) - if len(lines) > 1: - lines = [] - lines.extend(get_lines(opt_parts, indent)) - lines.extend(get_lines(pos_parts, indent)) - lines = [prog] + lines - - # join lines into usage - usage = '\n'.join(lines) - - # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) - - def _format_actions_usage(self, actions, groups): - # find group indices and identify actions in groups - group_actions = _set() - inserts = {} - for group in groups: - try: - start = actions.index(group._group_actions[0]) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: - for action in group._group_actions: - group_actions.add(action) - if not group.required: - inserts[start] = '[' - inserts[end] = ']' - else: - inserts[start] = '(' - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' - - # collect all actions format strings - parts = [] - for i, action in enumerate(actions): - - # suppressed arguments are marked with None - # remove | separators for suppressed arguments - if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) - - # produce all arg strings - elif not action.option_strings: - part = self._format_args(action, action.dest) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # add the action string to the list - parts.append(part) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = '%s' % option_string - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # insert things at the necessary indices - for i in _sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = _re.sub(r'\(([^|]*)\)', r'\1', text) - text = text.strip() - - # return the text - return text - - def _format_text(self, text): - if '%(prog)' in text: - text = text % dict(prog=self._prog) - text_width = self._width - self._current_indent - indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' - - def _format_action(self, action): - # determine the required width and the entry label - help_position = min(self._action_max_length + 2, - self._max_help_position) - help_width = self._width - help_position - action_width = help_position - self._current_indent - 2 - action_header = self._format_action_invocation(action) - - # ho nelp; start on same line and add a final newline - if not action.help: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - - # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header - action_header = '%*s%-*s ' % tup - indent_first = 0 - - # long action name; start on the next line - else: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - indent_first = help_position - - # collect the pieces of the action help - parts = [action_header] - - # if there was help for the action, add lines of help text - if action.help: - help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) - - # or add a newline if the description doesn't end with one - elif not action_header.endswith('\n'): - parts.append('\n') - - # if there are any sub-actions, add their help as well - for subaction in self._iter_indented_subactions(action): - parts.append(self._format_action(subaction)) - - # return a single string - return self._join_parts(parts) - - def _format_action_invocation(self, action): - if not action.option_strings: - metavar, = self._metavar_formatter(action, action.dest)(1) - return metavar - - else: - parts = [] - - # if the Optional doesn't take a value, format is: - # -s, --long - if action.nargs == 0: - parts.extend(action.option_strings) - - # if the Optional takes a value, format is: - # -s ARGS, --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) - - def _metavar_formatter(self, action, default_metavar): - if action.metavar is not None: - result = action.metavar - elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) - else: - result = default_metavar - - def format(tuple_size): - if isinstance(result, tuple): - return result - else: - return (result, ) * tuple_size - return format - - def _format_args(self, action, default_metavar): - get_metavar = self._metavar_formatter(action, default_metavar) - if action.nargs is None: - result = '%s' % get_metavar(1) - elif action.nargs == OPTIONAL: - result = '[%s]' % get_metavar(1) - elif action.nargs == ZERO_OR_MORE: - result = '[%s [%s ...]]' % get_metavar(2) - elif action.nargs == ONE_OR_MORE: - result = '%s [%s ...]' % get_metavar(2) - elif action.nargs == REMAINDER: - result = '...' - elif action.nargs == PARSER: - result = '%s ...' % get_metavar(1) - else: - formats = ['%s' for _ in range(action.nargs)] - result = ' '.join(formats) % get_metavar(action.nargs) - return result - - def _expand_help(self, action): - params = dict(vars(action), prog=self._prog) - for name in list(params): - if params[name] is SUPPRESS: - del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ - if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params - - def _iter_indented_subactions(self, action): - try: - get_subactions = action._get_subactions - except AttributeError: - pass - else: - self._indent() - for subaction in get_subactions(): - yield subaction - self._dedent() - - def _split_lines(self, text, width): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.wrap(text, width) - - def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.fill(text, width, initial_indent=indent, - subsequent_indent=indent) - - def _get_help_string(self, action): - return action.help - - -class RawDescriptionHelpFormatter(HelpFormatter): - """Help message formatter which retains any formatting in descriptions. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _fill_text(self, text, width, indent): - return ''.join([indent + line for line in text.splitlines(True)]) - - -class RawTextHelpFormatter(RawDescriptionHelpFormatter): - """Help message formatter which retains formatting of all help text. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _split_lines(self, text, width): - return text.splitlines() - - -class ArgumentDefaultsHelpFormatter(HelpFormatter): - """Help message formatter which adds default values to argument help. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _get_help_string(self, action): - help = action.help - if '%(default)' not in action.help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' - return help - - -# ===================== -# Options and Arguments -# ===================== - -def _get_action_name(argument): - if argument is None: - return None - elif argument.option_strings: - return '/'.join(argument.option_strings) - elif argument.metavar not in (None, SUPPRESS): - return argument.metavar - elif argument.dest not in (None, SUPPRESS): - return argument.dest - else: - return None - - -class ArgumentError(Exception): - """An error from creating or using an argument (optional or positional). - - The string value of this exception is the message, augmented with - information about the argument that caused it. - """ - - def __init__(self, argument, message): - self.argument_name = _get_action_name(argument) - self.message = message - - def __str__(self): - if self.argument_name is None: - format = '%(message)s' - else: - format = 'argument %(argument_name)s: %(message)s' - return format % dict(message=self.message, - argument_name=self.argument_name) - - -class ArgumentTypeError(Exception): - """An error from trying to convert a command line string to a type.""" - pass - - -# ============== -# Action classes -# ============== - -class Action(_AttributeHolder): - """Information about how to convert command line strings to Python objects. - - Action objects are used by an ArgumentParser to represent the information - needed to parse a single argument from one or more strings from the - command line. The keyword arguments to the Action constructor are also - all attributes of Action instances. - - Keyword Arguments: - - - option_strings -- A list of command-line option strings which - should be associated with this action. - - - dest -- The name of the attribute to hold the created object(s) - - - nargs -- The number of command-line arguments that should be - consumed. By default, one argument will be consumed and a single - value will be produced. Other values include: - - N (an integer) consumes N arguments (and produces a list) - - '?' consumes zero or one arguments - - '*' consumes zero or more arguments (and produces a list) - - '+' consumes one or more arguments (and produces a list) - Note that the difference between the default and nargs=1 is that - with the default, a single value will be produced, while with - nargs=1, a list containing a single value will be produced. - - - const -- The value to be produced if the option is specified and the - option uses an action that takes no values. - - - default -- The value to be produced if the option is not specified. - - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. - - - choices -- A container of values that should be allowed. If not None, - after a command-line argument has been converted to the appropriate - type, an exception will be raised if it is not a member of this - collection. - - - required -- True if the action must always be specified at the - command line. This is only meaningful for optional command-line - arguments. - - - help -- The help string describing the argument. - - - metavar -- The name to be used for the option's argument with the - help string. If None, the 'dest' value will be used as the name. - """ - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - self.option_strings = option_strings - self.dest = dest - self.nargs = nargs - self.const = const - self.default = default - self.type = type - self.choices = choices - self.required = required - self.help = help - self.metavar = metavar - - def _get_kwargs(self): - names = [ - 'option_strings', - 'dest', - 'nargs', - 'const', - 'default', - 'type', - 'choices', - 'help', - 'metavar', - ] - return [(name, getattr(self, name)) for name in names] - - def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - - -class _StoreAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for store actions must be > 0; if you ' - 'have nothing to store, actions such as store ' - 'true or store const may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_StoreAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - - -class _StoreConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_StoreConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.const) - - -class _StoreTrueAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=False, - required=False, - help=None): - super(_StoreTrueAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=True, - default=default, - required=required, - help=help) - - -class _StoreFalseAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=True, - required=False, - help=None): - super(_StoreFalseAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=False, - default=default, - required=required, - help=help) - - -class _AppendAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for append actions must be > 0; if arg ' - 'strings are not supplying the value to append, ' - 'the append const action may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_AppendAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(values) - setattr(namespace, self.dest, items) - - -class _AppendConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_AppendConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(self.const) - setattr(namespace, self.dest, items) - - -class _CountAction(Action): - - def __init__(self, - option_strings, - dest, - default=None, - required=False, - help=None): - super(_CountAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - new_count = _ensure_value(namespace, self.dest, 0) + 1 - setattr(namespace, self.dest, new_count) - - -class _HelpAction(Action): - - def __init__(self, - option_strings, - dest=SUPPRESS, - default=SUPPRESS, - help=None): - super(_HelpAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - parser.print_help() - parser.exit() - - -class _VersionAction(Action): - - def __init__(self, - option_strings, - version=None, - dest=SUPPRESS, - default=SUPPRESS, - help=None): - super(_VersionAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - self.version = version - - def __call__(self, parser, namespace, values, option_string=None): - version = self.version - if version is None: - version = parser.version - formatter = parser._get_formatter() - formatter.add_text(version) - parser.exit(message=formatter.format_help()) - - -class _SubParsersAction(Action): - - class _ChoicesPseudoAction(Action): - - def __init__(self, name, help): - sup = super(_SubParsersAction._ChoicesPseudoAction, self) - sup.__init__(option_strings=[], dest=name, help=help) - - def __init__(self, - option_strings, - prog, - parser_class, - dest=SUPPRESS, - help=None, - metavar=None): - - self._prog_prefix = prog - self._parser_class = parser_class - self._name_parser_map = {} - self._choices_actions = [] - - super(_SubParsersAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=PARSER, - choices=self._name_parser_map, - help=help, - metavar=metavar) - - def add_parser(self, name, **kwargs): - # set prog from the existing prefix - if kwargs.get('prog') is None: - kwargs['prog'] = '%s %s' % (self._prog_prefix, name) - - # create a pseudo-action to hold the choice help - if 'help' in kwargs: - help = kwargs.pop('help') - choice_action = self._ChoicesPseudoAction(name, help) - self._choices_actions.append(choice_action) - - # create the parser and add it to the map - parser = self._parser_class(**kwargs) - self._name_parser_map[name] = parser - return parser - - def _get_subactions(self): - return self._choices_actions - - def __call__(self, parser, namespace, values, option_string=None): - parser_name = values[0] - arg_strings = values[1:] - - # set the parser name if requested - if self.dest is not SUPPRESS: - setattr(namespace, self.dest, parser_name) - - # select the parser - try: - parser = self._name_parser_map[parser_name] - except KeyError: - tup = parser_name, ', '.join(self._name_parser_map) - msg = _('unknown parser %r (choices: %s)' % tup) - raise ArgumentError(self, msg) - - # parse all the remaining options into the namespace - parser.parse_args(arg_strings, namespace) - - -# ============== -# Type classes -# ============== - -class FileType(object): - """Factory for creating file object types - - Instances of FileType are typically passed as type= arguments to the - ArgumentParser add_argument() method. - - Keyword Arguments: - - mode -- A string indicating how the file is to be opened. Accepts the - same values as the builtin open() function. - - bufsize -- The file's desired buffer size. Accepts the same values as - the builtin open() function. - """ - - def __init__(self, mode='r', bufsize=None): - self._mode = mode - self._bufsize = bufsize - - def __call__(self, string): - # the special argument "-" means sys.std{in,out} - if string == '-': - if 'r' in self._mode: - return _sys.stdin - elif 'w' in self._mode: - return _sys.stdout - else: - msg = _('argument "-" with mode %r' % self._mode) - raise ValueError(msg) - - # all other arguments are used as file names - if self._bufsize: - return open(string, self._mode, self._bufsize) - else: - return open(string, self._mode) - - def __repr__(self): - args = [self._mode, self._bufsize] - args_str = ', '.join([repr(arg) for arg in args if arg is not None]) - return '%s(%s)' % (type(self).__name__, args_str) - -# =========================== -# Optional and Positional Parsing -# =========================== - -class Namespace(_AttributeHolder): - """Simple object for storing attributes. - - Implements equality by attribute names and values, and provides a simple - string representation. - """ - - def __init__(self, **kwargs): - for name in kwargs: - setattr(self, name, kwargs[name]) - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not (self == other) - - def __contains__(self, key): - return key in self.__dict__ - - -class _ActionsContainer(object): - - def __init__(self, - description, - prefix_chars, - argument_default, - conflict_handler): - super(_ActionsContainer, self).__init__() - - self.description = description - self.argument_default = argument_default - self.prefix_chars = prefix_chars - self.conflict_handler = conflict_handler - - # set up registries - self._registries = {} - - # register actions - self.register('action', None, _StoreAction) - self.register('action', 'store', _StoreAction) - self.register('action', 'store_const', _StoreConstAction) - self.register('action', 'store_true', _StoreTrueAction) - self.register('action', 'store_false', _StoreFalseAction) - self.register('action', 'append', _AppendAction) - self.register('action', 'append_const', _AppendConstAction) - self.register('action', 'count', _CountAction) - self.register('action', 'help', _HelpAction) - self.register('action', 'version', _VersionAction) - self.register('action', 'parsers', _SubParsersAction) - - # raise an exception if the conflict handler is invalid - self._get_handler() - - # action storage - self._actions = [] - self._option_string_actions = {} - - # groups - self._action_groups = [] - self._mutually_exclusive_groups = [] - - # defaults storage - self._defaults = {} - - # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') - - # whether or not there are any optionals that look like negative - # numbers -- uses a list so it can be shared and edited - self._has_negative_number_optionals = [] - - # ==================== - # Registration methods - # ==================== - def register(self, registry_name, value, object): - registry = self._registries.setdefault(registry_name, {}) - registry[value] = object - - def _registry_get(self, registry_name, value, default=None): - return self._registries[registry_name].get(value, default) - - # ================================== - # Namespace default accessor methods - # ================================== - def set_defaults(self, **kwargs): - self._defaults.update(kwargs) - - # if these defaults match any existing arguments, replace - # the previous default on the object with the new one - for action in self._actions: - if action.dest in kwargs: - action.default = kwargs[action.dest] - - def get_default(self, dest): - for action in self._actions: - if action.dest == dest and action.default is not None: - return action.default - return self._defaults.get(dest, None) - - - # ======================= - # Adding argument actions - # ======================= - def add_argument(self, *args, **kwargs): - """ - add_argument(dest, ..., name=value, ...) - add_argument(option_string, option_string, ..., name=value, ...) - """ - - # if no positional args are supplied or only one is supplied and - # it doesn't look like an option string, parse a positional - # argument - chars = self.prefix_chars - if not args or len(args) == 1 and args[0][0] not in chars: - if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') - kwargs = self._get_positional_kwargs(*args, **kwargs) - - # otherwise, we're adding an optional argument - else: - kwargs = self._get_optional_kwargs(*args, **kwargs) - - # if no default was supplied, use the parser-level default - if 'default' not in kwargs: - dest = kwargs['dest'] - if dest in self._defaults: - kwargs['default'] = self._defaults[dest] - elif self.argument_default is not None: - kwargs['default'] = self.argument_default - - # create the action object, and add it to the parser - action_class = self._pop_action_class(kwargs) - if not _callable(action_class): - raise ValueError('unknown action "%s"' % action_class) - action = action_class(**kwargs) - - # raise an error if the action type is not callable - type_func = self._registry_get('type', action.type, action.type) - if not _callable(type_func): - raise ValueError('%r is not callable' % type_func) - - return self._add_action(action) - - def add_argument_group(self, *args, **kwargs): - group = _ArgumentGroup(self, *args, **kwargs) - self._action_groups.append(group) - return group - - def add_mutually_exclusive_group(self, **kwargs): - group = _MutuallyExclusiveGroup(self, **kwargs) - self._mutually_exclusive_groups.append(group) - return group - - def _add_action(self, action): - # resolve any conflicts - self._check_conflict(action) - - # add to actions list - self._actions.append(action) - action.container = self - - # index the action by any option strings it has - for option_string in action.option_strings: - self._option_string_actions[option_string] = action - - # set the flag if any option strings look like negative numbers - for option_string in action.option_strings: - if self._negative_number_matcher.match(option_string): - if not self._has_negative_number_optionals: - self._has_negative_number_optionals.append(True) - - # return the created action - return action - - def _remove_action(self, action): - self._actions.remove(action) - - def _add_container_actions(self, container): - # collect groups by titles - title_group_map = {} - for group in self._action_groups: - if group.title in title_group_map: - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) - title_group_map[group.title] = group - - # map each action to its group - group_map = {} - for group in container._action_groups: - - # if a group with the title exists, use that, otherwise - # create a new group matching the container's group - if group.title not in title_group_map: - title_group_map[group.title] = self.add_argument_group( - title=group.title, - description=group.description, - conflict_handler=group.conflict_handler) - - # map the actions to their new group - for action in group._group_actions: - group_map[action] = title_group_map[group.title] - - # add container's mutually exclusive groups - # NOTE: if add_mutually_exclusive_group ever gains title= and - # description= then this code will need to be expanded as above - for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( - required=group.required) - - # map the actions to their new mutex group - for action in group._group_actions: - group_map[action] = mutex_group - - # add all actions to this container or their group - for action in container._actions: - group_map.get(action, self)._add_action(action) - - def _get_positional_kwargs(self, dest, **kwargs): - # make sure required is not specified - if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") - raise TypeError(msg) - - # mark positional arguments as required if at least one is - # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: - kwargs['required'] = True - - # return the keyword arguments with no option strings - return dict(kwargs, dest=dest, option_strings=[]) - - def _get_optional_kwargs(self, *args, **kwargs): - # determine short and long option strings - option_strings = [] - long_option_strings = [] - for option_string in args: - # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: - msg = _('invalid option string %r: ' - 'must start with a character %r') - tup = option_string, self.prefix_chars - raise ValueError(msg % tup) - - # strings starting with two prefix characters are long options - option_strings.append(option_string) - if option_string[0] in self.prefix_chars: - if len(option_string) > 1: - if option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) - - # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' - dest = kwargs.pop('dest', None) - if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) - if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) - dest = dest.replace('-', '_') - - # return the updated keyword arguments - return dict(kwargs, dest=dest, option_strings=option_strings) - - def _pop_action_class(self, kwargs, default=None): - action = kwargs.pop('action', default) - return self._registry_get('action', action, action) - - def _get_handler(self): - # determine function from conflict handler string - handler_func_name = '_handle_conflict_%s' % self.conflict_handler - try: - return getattr(self, handler_func_name) - except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) - - def _check_conflict(self, action): - - # find all options that conflict with this option - confl_optionals = [] - for option_string in action.option_strings: - if option_string in self._option_string_actions: - confl_optional = self._option_string_actions[option_string] - confl_optionals.append((option_string, confl_optional)) - - # resolve any conflicts - if confl_optionals: - conflict_handler = self._get_handler() - conflict_handler(action, confl_optionals) - - def _handle_conflict_error(self, action, conflicting_actions): - message = _('conflicting option string(s): %s') - conflict_string = ', '.join([option_string - for option_string, action - in conflicting_actions]) - raise ArgumentError(action, message % conflict_string) - - def _handle_conflict_resolve(self, action, conflicting_actions): - - # remove all conflicting options - for option_string, action in conflicting_actions: - - # remove the conflicting option - action.option_strings.remove(option_string) - self._option_string_actions.pop(option_string, None) - - # if the option now has no option string, remove it from the - # container holding it - if not action.option_strings: - action.container._remove_action(action) - - -class _ArgumentGroup(_ActionsContainer): - - def __init__(self, container, title=None, description=None, **kwargs): - # add any missing keyword arguments by checking the container - update = kwargs.setdefault - update('conflict_handler', container.conflict_handler) - update('prefix_chars', container.prefix_chars) - update('argument_default', container.argument_default) - super_init = super(_ArgumentGroup, self).__init__ - super_init(description=description, **kwargs) - - # group attributes - self.title = title - self._group_actions = [] - - # share most attributes with the container - self._registries = container._registries - self._actions = container._actions - self._option_string_actions = container._option_string_actions - self._defaults = container._defaults - self._has_negative_number_optionals = \ - container._has_negative_number_optionals - - def _add_action(self, action): - action = super(_ArgumentGroup, self)._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - super(_ArgumentGroup, self)._remove_action(action) - self._group_actions.remove(action) - - -class _MutuallyExclusiveGroup(_ArgumentGroup): - - def __init__(self, container, required=False): - super(_MutuallyExclusiveGroup, self).__init__(container) - self.required = required - self._container = container - - def _add_action(self, action): - if action.required: - msg = _('mutually exclusive arguments must be optional') - raise ValueError(msg) - action = self._container._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - self._container._remove_action(action) - self._group_actions.remove(action) - - -class ArgumentParser(_AttributeHolder, _ActionsContainer): - """Object for parsing command line strings into Python objects. - - Keyword Arguments: - - prog -- The name of the program (default: sys.argv[0]) - - usage -- A usage message (default: auto-generated from arguments) - - description -- A description of what the program does - - epilog -- Text following the argument descriptions - - parents -- Parsers whose arguments should be copied into this one - - formatter_class -- HelpFormatter class for printing help messages - - prefix_chars -- Characters that prefix optional arguments - - fromfile_prefix_chars -- Characters that prefix files containing - additional arguments - - argument_default -- The default value for all arguments - - conflict_handler -- String indicating how to handle conflicts - - add_help -- Add a -h/-help option - """ - - def __init__(self, - prog=None, - usage=None, - description=None, - epilog=None, - version=None, - parents=[], - formatter_class=HelpFormatter, - prefix_chars='-', - fromfile_prefix_chars=None, - argument_default=None, - conflict_handler='error', - add_help=True): - - if version is not None: - import warnings - warnings.warn( - """The "version" argument to ArgumentParser is deprecated. """ - """Please use """ - """"add_argument(..., action='version', version="N", ...)" """ - """instead""", DeprecationWarning) - - superinit = super(ArgumentParser, self).__init__ - superinit(description=description, - prefix_chars=prefix_chars, - argument_default=argument_default, - conflict_handler=conflict_handler) - - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog - self.usage = usage - self.epilog = epilog - self.version = version - self.formatter_class = formatter_class - self.fromfile_prefix_chars = fromfile_prefix_chars - self.add_help = add_help - - add_group = self.add_argument_group - self._positionals = add_group(_('positional arguments')) - self._optionals = add_group(_('optional arguments')) - self._subparsers = None - - # register types - def identity(string): - return string - self.register('type', None, identity) - - # add help and version arguments if necessary - # (using explicit default to override global argument_default) - if self.add_help: - self.add_argument( - '-h', '--help', action='help', default=SUPPRESS, - help=_('show this help message and exit')) - if self.version: - self.add_argument( - '-v', '--version', action='version', default=SUPPRESS, - version=self.version, - help=_("show program's version number and exit")) - - # add parent arguments and defaults - for parent in parents: - self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) - - # ======================= - # Pretty __repr__ methods - # ======================= - def _get_kwargs(self): - names = [ - 'prog', - 'usage', - 'description', - 'version', - 'formatter_class', - 'conflict_handler', - 'add_help', - ] - return [(name, getattr(self, name)) for name in names] - - # ================================== - # Optional/Positional adding methods - # ================================== - def add_subparsers(self, **kwargs): - if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) - - # add the parser class to the arguments if it's not present - kwargs.setdefault('parser_class', type(self)) - - if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) - self._subparsers = self.add_argument_group(title, description) - else: - self._subparsers = self._positionals - - # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix - if kwargs.get('prog') is None: - formatter = self._get_formatter() - positionals = self._get_positional_actions() - groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') - kwargs['prog'] = formatter.format_help().strip() - - # create the parsers action and add it to the positionals list - parsers_class = self._pop_action_class(kwargs, 'parsers') - action = parsers_class(option_strings=[], **kwargs) - self._subparsers._add_action(action) - - # return the created parsers action - return action - - def _add_action(self, action): - if action.option_strings: - self._optionals._add_action(action) - else: - self._positionals._add_action(action) - return action - - def _get_optional_actions(self): - return [action - for action in self._actions - if action.option_strings] - - def _get_positional_actions(self): - return [action - for action in self._actions - if not action.option_strings] - - # ===================================== - # Command line argument parsing methods - # ===================================== - def parse_args(self, args=None, namespace=None): - args, argv = self.parse_known_args(args, namespace) - if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) - return args - - def parse_known_args(self, args=None, namespace=None): - # args default to the system args - if args is None: - args = _sys.argv[1:] - - # default Namespace built from parser defaults - if namespace is None: - namespace = Namespace() - - # add any action defaults that aren't present - for action in self._actions: - if action.dest is not SUPPRESS: - if not hasattr(namespace, action.dest): - if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, _basestring): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) - - # add any parser defaults that aren't present - for dest in self._defaults: - if not hasattr(namespace, dest): - setattr(namespace, dest, self._defaults[dest]) - - # parse the arguments and exit if there are any errors - try: - return self._parse_known_args(args, namespace) - except ArgumentError: - err = _sys.exc_info()[1] - self.error(str(err)) - - def _parse_known_args(self, arg_strings, namespace): - # replace arg strings that are file references - if self.fromfile_prefix_chars is not None: - arg_strings = self._read_args_from_files(arg_strings) - - # map all mutually exclusive arguments to the other arguments - # they can't occur with - action_conflicts = {} - for mutex_group in self._mutually_exclusive_groups: - group_actions = mutex_group._group_actions - for i, mutex_action in enumerate(mutex_group._group_actions): - conflicts = action_conflicts.setdefault(mutex_action, []) - conflicts.extend(group_actions[:i]) - conflicts.extend(group_actions[i + 1:]) - - # find all option indices, and determine the arg_string_pattern - # which has an 'O' if there is an option at an index, - # an 'A' if there is an argument, or a '-' if there is a '--' - option_string_indices = {} - arg_string_pattern_parts = [] - arg_strings_iter = iter(arg_strings) - for i, arg_string in enumerate(arg_strings_iter): - - # all args after -- are non-options - if arg_string == '--': - arg_string_pattern_parts.append('-') - for arg_string in arg_strings_iter: - arg_string_pattern_parts.append('A') - - # otherwise, add the arg to the arg strings - # and note the index if it was an option - else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: - pattern = 'A' - else: - option_string_indices[i] = option_tuple - pattern = 'O' - arg_string_pattern_parts.append(pattern) - - # join the pieces together to form the pattern - arg_strings_pattern = ''.join(arg_string_pattern_parts) - - # converts arg strings to the appropriate and then takes the action - seen_actions = _set() - seen_non_default_actions = _set() - - def take_action(action, argument_strings, option_string=None): - seen_actions.add(action) - argument_values = self._get_values(action, argument_strings) - - # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: - seen_non_default_actions.add(action) - for conflict_action in action_conflicts.get(action, []): - if conflict_action in seen_non_default_actions: - msg = _('not allowed with argument %s') - action_name = _get_action_name(conflict_action) - raise ArgumentError(action, msg % action_name) - - # take the action if we didn't receive a SUPPRESS value - # (e.g. from a default) - if argument_values is not SUPPRESS: - action(self, namespace, argument_values, option_string) - - # function to convert arg_strings into an optional action - def consume_optional(start_index): - - # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple - - # identify additional optionals in the same arg string - # (e.g. -xyz is the same as -x -y -z if no args are required) - match_argument = self._match_argument - action_tuples = [] - while True: - - # if we found no optional action, skip it - if action is None: - extras.append(arg_strings[start_index]) - return start_index + 1 - - # if there is an explicit argument, try to match the - # optional's string arguments to only this - if explicit_arg is not None: - arg_count = match_argument(action, 'A') - - # if the action is a single-dash option and takes no - # arguments, try to parse more single-dash options out - # of the tail of the option string - chars = self.prefix_chars - if arg_count == 0 and option_string[1] not in chars: - action_tuples.append((action, [], option_string)) - for char in self.prefix_chars: - option_string = char + explicit_arg[0] - explicit_arg = explicit_arg[1:] or None - optionals_map = self._option_string_actions - if option_string in optionals_map: - action = optionals_map[option_string] - break - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if the action expect exactly one argument, we've - # successfully matched the option; exit the loop - elif arg_count == 1: - stop = start_index + 1 - args = [explicit_arg] - action_tuples.append((action, args, option_string)) - break - - # error if a double-dash option did not use the - # explicit argument - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if there is no explicit argument, try to match the - # optional's string arguments with the following strings - # if successful, exit the loop - else: - start = start_index + 1 - selected_patterns = arg_strings_pattern[start:] - arg_count = match_argument(action, selected_patterns) - stop = start + arg_count - args = arg_strings[start:stop] - action_tuples.append((action, args, option_string)) - break - - # add the Optional to the list and return the index at which - # the Optional's string args stopped - assert action_tuples - for action, args, option_string in action_tuples: - take_action(action, args, option_string) - return stop - - # the list of Positionals left to be parsed; this is modified - # by consume_positionals() - positionals = self._get_positional_actions() - - # function to convert arg_strings into positional actions - def consume_positionals(start_index): - # match as many Positionals as possible - match_partial = self._match_arguments_partial - selected_pattern = arg_strings_pattern[start_index:] - arg_counts = match_partial(positionals, selected_pattern) - - # slice off the appropriate arg strings for each Positional - # and add the Positional and its args to the list - for action, arg_count in zip(positionals, arg_counts): - args = arg_strings[start_index: start_index + arg_count] - start_index += arg_count - take_action(action, args) - - # slice off the Positionals that we just parsed and return the - # index at which the Positionals' string args stopped - positionals[:] = positionals[len(arg_counts):] - return start_index - - # consume Positionals and Optionals alternately, until we have - # passed the last option string - extras = [] - start_index = 0 - if option_string_indices: - max_option_string_index = max(option_string_indices) - else: - max_option_string_index = -1 - while start_index <= max_option_string_index: - - # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: - positionals_end_index = consume_positionals(start_index) - - # only try to parse the next optional if we didn't consume - # the option string during the positionals parsing - if positionals_end_index > start_index: - start_index = positionals_end_index - continue - else: - start_index = positionals_end_index - - # if we consumed all the positionals we could and we're not - # at the index of an option string, there were extra arguments - if start_index not in option_string_indices: - strings = arg_strings[start_index:next_option_string_index] - extras.extend(strings) - start_index = next_option_string_index - - # consume the next optional and any arguments for it - start_index = consume_optional(start_index) - - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) - - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) - - # if we didn't use all the Positional objects, there were too few - # arg strings supplied. - if positionals: - self.error(_('too few arguments')) - - # make sure all required actions were present - for action in self._actions: - if action.required: - if action not in seen_actions: - name = _get_action_name(action) - self.error(_('argument %s is required') % name) - - # make sure all required groups had one option present - for group in self._mutually_exclusive_groups: - if group.required: - for action in group._group_actions: - if action in seen_non_default_actions: - break - - # if no actions were used, report the error - else: - names = [_get_action_name(action) - for action in group._group_actions - if action.help is not SUPPRESS] - msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) - - # return the updated namespace and the extra arguments - return namespace, extras - - def _read_args_from_files(self, arg_strings): - # expand arguments referencing files - new_arg_strings = [] - for arg_string in arg_strings: - - # for regular arguments, just add them back into the list - if arg_string[0] not in self.fromfile_prefix_chars: - new_arg_strings.append(arg_string) - - # replace arguments referencing files with the file content - else: - try: - args_file = open(arg_string[1:]) - try: - arg_strings = [] - for arg_line in args_file.read().splitlines(): - for arg in self.convert_arg_line_to_args(arg_line): - arg_strings.append(arg) - arg_strings = self._read_args_from_files(arg_strings) - new_arg_strings.extend(arg_strings) - finally: - args_file.close() - except IOError: - err = _sys.exc_info()[1] - self.error(str(err)) - - # return the modified argument list - return new_arg_strings - - def convert_arg_line_to_args(self, arg_line): - return [arg_line] - - def _match_argument(self, action, arg_strings_pattern): - # match the pattern for this action to the arg strings - nargs_pattern = self._get_nargs_pattern(action) - match = _re.match(nargs_pattern, arg_strings_pattern) - - # raise an exception if we weren't able to find a match - if match is None: - nargs_errors = { - None: _('expected one argument'), - OPTIONAL: _('expected at most one argument'), - ONE_OR_MORE: _('expected at least one argument'), - } - default = _('expected %s argument(s)') % action.nargs - msg = nargs_errors.get(action.nargs, default) - raise ArgumentError(action, msg) - - # return the number of arguments matched - return len(match.group(1)) - - def _match_arguments_partial(self, actions, arg_strings_pattern): - # progressively shorten the actions list by slicing off the - # final actions until we find a match - result = [] - for i in range(len(actions), 0, -1): - actions_slice = actions[:i] - pattern = ''.join([self._get_nargs_pattern(action) - for action in actions_slice]) - match = _re.match(pattern, arg_strings_pattern) - if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result - - def _parse_optional(self, arg_string): - # if it's an empty string, it was meant to be a positional - if not arg_string: - return None - - # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: - return None - - # if the option string is present in the parser, return the action - if arg_string in self._option_string_actions: - action = self._option_string_actions[arg_string] - return action, arg_string, None - - # if it's just a single character, it was meant to be positional - if len(arg_string) == 1: - return None - - # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg - - # search through all possible prefixes of the option string - # and all actions in the parser for possible interpretations - option_tuples = self._get_option_tuples(arg_string) - - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - tup = arg_string, options - self.error(_('ambiguous option: %s could match %s') % tup) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple - - # if it was not found as an option, but it looks like a negative - # number, it was meant to be positional - # unless there are negative-number-like options - if self._negative_number_matcher.match(arg_string): - if not self._has_negative_number_optionals: - return None - - # if it contains a space, it was meant to be a positional - if ' ' in arg_string: - return None - - # it was meant to be an optional but there is no such option - # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None - - def _get_option_tuples(self, option_string): - result = [] - - # option strings starting with two prefix characters are only - # split at the '=' - chars = self.prefix_chars - if option_string[0] in chars and option_string[1] in chars: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None - for option_string in self._option_string_actions: - if option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # single character options can be concatenated with their arguments - # but multiple character options always have to have their argument - # separate - elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string - explicit_arg = None - short_option_prefix = option_string[:2] - short_explicit_arg = option_string[2:] - - for option_string in self._option_string_actions: - if option_string == short_option_prefix: - action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg - result.append(tup) - elif option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # shouldn't ever get here - else: - self.error(_('unexpected option string: %s') % option_string) - - # return the collected option tuples - return result - - def _get_nargs_pattern(self, action): - # in all examples below, we have to allow for '--' args - # which are represented as '-' in the pattern - nargs = action.nargs - - # the default (None) is assumed to be a single argument - if nargs is None: - nargs_pattern = '(-*A-*)' - - # allow zero or one arguments - elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' - - # allow zero or more arguments - elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' - - # allow one or more arguments - elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' - - # allow any number of options or arguments - elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' - - # allow one argument followed by any number of options or arguments - elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' - - # all others should be integers - else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') - - # return the pattern - return nargs_pattern - - # ======================== - # Value conversion methods - # ======================== - def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' - if action.nargs not in [PARSER, REMAINDER]: - arg_strings = [s for s in arg_strings if s != '--'] - - # optional argument produces a default when not present - if not arg_strings and action.nargs == OPTIONAL: - if action.option_strings: - value = action.const - else: - value = action.default - if isinstance(value, _basestring): - value = self._get_value(action, value) - self._check_value(action, value) - - # when nargs='*' on a positional, if there were no command-line - # args, use the default if it is anything other than None - elif (not arg_strings and action.nargs == ZERO_OR_MORE and - not action.option_strings): - if action.default is not None: - value = action.default - else: - value = arg_strings - self._check_value(action, value) - - # single argument or optional argument produces a single value - elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: - arg_string, = arg_strings - value = self._get_value(action, arg_string) - self._check_value(action, value) - - # REMAINDER arguments convert all values, checking none - elif action.nargs == REMAINDER: - value = [self._get_value(action, v) for v in arg_strings] - - # PARSER arguments convert all values, but check only the first - elif action.nargs == PARSER: - value = [self._get_value(action, v) for v in arg_strings] - self._check_value(action, value[0]) - - # all other types of nargs produce a list - else: - value = [self._get_value(action, v) for v in arg_strings] - for v in value: - self._check_value(action, v) - - # return the converted value - return value - - def _get_value(self, action, arg_string): - type_func = self._registry_get('type', action.type, action.type) - if not _callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) - - # convert the value to the appropriate type - try: - result = type_func(arg_string) - - # ArgumentTypeErrors indicate errors - except ArgumentTypeError: - name = getattr(action.type, '__name__', repr(action.type)) - msg = str(_sys.exc_info()[1]) - raise ArgumentError(action, msg) - - # TypeErrors or ValueErrors also indicate errors - except (TypeError, ValueError): - name = getattr(action.type, '__name__', repr(action.type)) - msg = _('invalid %s value: %r') - raise ArgumentError(action, msg % (name, arg_string)) - - # return the converted value - return result - - def _check_value(self, action, value): - # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - tup = value, ', '.join(map(repr, action.choices)) - msg = _('invalid choice: %r (choose from %s)') % tup - raise ArgumentError(action, msg) - - # ======================= - # Help-formatting methods - # ======================= - def format_usage(self): - formatter = self._get_formatter() - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - return formatter.format_help() - - def format_help(self): - formatter = self._get_formatter() - - # usage - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - - # description - formatter.add_text(self.description) - - # positionals, optionals and user-defined groups - for action_group in self._action_groups: - formatter.start_section(action_group.title) - formatter.add_text(action_group.description) - formatter.add_arguments(action_group._group_actions) - formatter.end_section() - - # epilog - formatter.add_text(self.epilog) - - # determine help from format above - return formatter.format_help() - - def format_version(self): - import warnings - warnings.warn( - 'The format_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - formatter = self._get_formatter() - formatter.add_text(self.version) - return formatter.format_help() - - def _get_formatter(self): - return self.formatter_class(prog=self.prog) - - # ===================== - # Help-printing methods - # ===================== - def print_usage(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_usage(), file) - - def print_help(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_help(), file) - - def print_version(self, file=None): - import warnings - warnings.warn( - 'The print_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - self._print_message(self.format_version(), file) - - def _print_message(self, message, file=None): - if message: - if file is None: - file = _sys.stderr - file.write(message) - - # =============== - # Exiting methods - # =============== - def exit(self, status=0, message=None): - if message: - self._print_message(message, _sys.stderr) - _sys.exit(status) - - def error(self, message): - """error(message: string) - - Prints a usage message incorporating the message to stderr and - exits. - - If you override this in a subclass, it should not return -- it - should either exit or raise an exception. - """ - self.print_usage(_sys.stderr) - self.exit(2, _('%s: error: %s\n') % (self.prog, message)) diff --git a/IPython/external/mathjax.py b/IPython/external/mathjax.py index 2e499fe..d0d2ebc 100644 --- a/IPython/external/mathjax.py +++ b/IPython/external/mathjax.py @@ -19,6 +19,10 @@ From the command line: $ python -m IPython.external.mathjax +To a specific profile: + + $ python -m IPython.external.mathjax --profile=research + To install MathJax from a file you have already downloaded: $ python -m IPython.external.mathjax mathjax-xxx.tar.gz @@ -46,6 +50,7 @@ To find the directory where IPython would like MathJax installed: # Imports #----------------------------------------------------------------------------- +import argparse import os import shutil import sys @@ -55,7 +60,6 @@ import zipfile from IPython.utils.path import locate_profile -from IPython.external import argparse #----------------------------------------------------------------------------- # #----------------------------------------------------------------------------- @@ -200,20 +204,26 @@ def main() : ) parser.add_argument( + '-p', + '--profile', + default='default', + help='profile to install MathJax to (default is default)') + + parser.add_argument( '-i', '--install-dir', - default=default_dest, - help='installation directory (by default : %s)' % (default_dest)) + help='custom installation directory') + parser.add_argument( '-d', '--dest', action='store_true', - help='print where is current mathjax would be installed and exit') + help='print where current mathjax would be installed and exit') parser.add_argument( '-r', '--replace', action='store_true', - help='Wether to replace current mathjax if already exist') + help='Whether to replace current mathjax if it already exists') parser.add_argument( '-t', '--test', @@ -225,7 +235,13 @@ def main() : pargs = parser.parse_args() - dest = pargs.install_dir + if pargs.install_dir: + # Explicit install_dir overrides profile + dest = pargs.install_dir + else: + profile = pargs.profile + dest = os.path.join(locate_profile(profile), 'static', 'mathjax') + if pargs.dest : print dest return @@ -259,7 +275,7 @@ def main() : if __name__ == '__main__' : sys.exit(main()) -__all__ = ['install_mathjax', 'main', 'dest'] +__all__ = ['install_mathjax', 'main', 'default_dest'] """ Test notes: diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py index 08b4f58..955f6df 100644 --- a/IPython/html/notebookapp.py +++ b/IPython/html/notebookapp.py @@ -78,7 +78,7 @@ from IPython.kernel.zmq.kernelapp import ( kernel_aliases, ) from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils import submodule from IPython.utils.traitlets import ( Dict, Unicode, Integer, List, Bool, Bytes, @@ -253,7 +253,7 @@ aliases.update({ aliases.pop('f', None) notebook_aliases = [u'port', u'port-retries', u'ip', u'keyfile', u'certfile', - u'notebook-dir'] + u'notebook-dir', u'profile', u'profile-dir'] #----------------------------------------------------------------------------- # NotebookApp @@ -293,9 +293,11 @@ class NotebookApp(BaseIPythonApplication): # Network related information. - ip = Unicode(LOCALHOST, config=True, + ip = Unicode(config=True, help="The IP address the notebook server will listen on." ) + def _ip_default(self): + return localhost() def _ip_changed(self, name, old, new): if new == u'*': self.ip = u'' @@ -471,17 +473,10 @@ class NotebookApp(BaseIPythonApplication): help=("Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For headers" "sent by the upstream reverse proxy. Neccesary if the proxy handles SSL") ) - + def parse_command_line(self, argv=None): super(NotebookApp, self).parse_command_line(argv) - if argv is None: - argv = sys.argv[1:] - - # Scrub frontend-specific flags - self.kernel_argv = swallow_argv(argv, notebook_aliases, notebook_flags) - # Kernel should inherit default config file from frontend - self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) - + if self.extra_args: f = os.path.abspath(self.extra_args[0]) if os.path.isdir(f): @@ -491,6 +486,15 @@ class NotebookApp(BaseIPythonApplication): nbdir = os.path.dirname(f) self.config.NotebookManager.notebook_dir = nbdir + def init_kernel_argv(self): + """construct the kernel arguments""" + # Scrub frontend-specific flags + self.kernel_argv = swallow_argv(self.argv, notebook_aliases, notebook_flags) + # Kernel should inherit default config file from frontend + self.kernel_argv.append("--IPKernelApp.parent_appname='%s'" % self.name) + # Kernel should get *absolute* path to profile directory + self.kernel_argv.extend(["--profile-dir", self.profile_dir.location]) + def init_configurables(self): # force Session default to be secure default_secure(self.config) @@ -657,6 +661,7 @@ class NotebookApp(BaseIPythonApplication): def initialize(self, argv=None): self.init_logging() super(NotebookApp, self).initialize(argv) + self.init_kernel_argv() self.init_configurables() self.init_components() self.init_webapp() @@ -691,7 +696,7 @@ class NotebookApp(BaseIPythonApplication): info("Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).") if self.open_browser or self.file_to_run: - ip = self.ip or LOCALHOST + ip = self.ip or localhost() try: browser = webbrowser.get(self.browser or None) except webbrowser.Error as e: diff --git a/IPython/html/static/base/less/style.less b/IPython/html/static/base/less/style.less index 9981053..000adf0 100644 --- a/IPython/html/static/base/less/style.less +++ b/IPython/html/static/base/less/style.less @@ -1,4 +1,4 @@ @import "../base/less/variables.less"; @import "../base/less/mixins.less"; @import "../base/less/flexbox.less"; -@import "../base/less/page.less"; + diff --git a/IPython/html/static/custom/custom.js b/IPython/html/static/custom/custom.js index a276b2d..f8d1afe 100644 --- a/IPython/html/static/custom/custom.js +++ b/IPython/html/static/custom/custom.js @@ -17,12 +17,14 @@ * Create a custom button in toolbar that execute `%qtconsole` in kernel * and hence open a qtconsole attached to the same kernel as the current notebook * - * $([IPython.events]).on('notebook_loaded.Notebook', function(){ + * $([IPython.events]).on('app_initialized.NotebookApp', function(){ * IPython.toolbar.add_buttons_group([ * { * 'label' : 'run qtconsole', - * 'icon' : 'ui-icon-calculator', // select your icon from http://jqueryui.com/themeroller/ - * 'callback': function(){IPython.notebook.kernel.execute('%qtconsole')} + * 'icon' : 'icon-terminal', // select your icon from http://fortawesome.github.io/Font-Awesome/icons + * 'callback': function () { + * IPython.notebook.kernel.execute('%qtconsole') + * } * } * // add more button here if needed. * ]); diff --git a/IPython/html/static/notebook/js/cell.js b/IPython/html/static/notebook/js/cell.js index 1dbbf40..ec3daa0 100644 --- a/IPython/html/static/notebook/js/cell.js +++ b/IPython/html/static/notebook/js/cell.js @@ -207,7 +207,7 @@ var IPython = (function (IPython) { /** - * can the cell be splitted in 2 cells. + * can the cell be split into two cells * @method is_splittable **/ Cell.prototype.is_splittable = function () { @@ -216,6 +216,15 @@ var IPython = (function (IPython) { /** + * can the cell be merged with other cells + * @method is_mergeable + **/ + Cell.prototype.is_mergeable = function () { + return true; + }; + + + /** * @return {String} - the text before the cursor * @method get_pre_cursor **/ @@ -323,8 +332,13 @@ var IPython = (function (IPython) { } } } - // fallback on default (python) - var default_mode = this.default_mode || 'text/plain'; + // fallback on default + var default_mode + try { + default_mode = this._options.cm_config.mode; + } catch(e) { + default_mode = 'text/plain'; + } this.code_mirror.setOption('mode', default_mode); }; diff --git a/IPython/html/static/notebook/js/codecell.js b/IPython/html/static/notebook/js/codecell.js index 421bbbe..32ea289 100644 --- a/IPython/html/static/notebook/js/codecell.js +++ b/IPython/html/static/notebook/js/codecell.js @@ -65,8 +65,8 @@ var IPython = (function (IPython) { this.code_mirror = null; this.input_prompt_number = null; this.collapsed = false; - this.default_mode = 'ipython'; this.cell_type = "code"; + this.last_msg_id = null; var cm_overwrite_options = { @@ -167,7 +167,7 @@ var IPython = (function (IPython) { // triger on keypress (!) otherwise inconsistent event.which depending on plateform // browser and keyboard layout ! // Pressing '(' , request tooltip, don't forget to reappend it - // The second argument says to hide the tooltip if the docstring + // The second argument says to hide the tooltip if the docstring // is actually empty IPython.tooltip.pending(that, true); } else if (event.which === key.UPARROW && event.type === 'keydown') { @@ -180,8 +180,7 @@ var IPython = (function (IPython) { return true; }; } else if (event.which === key.ESC) { - IPython.tooltip.remove_and_cancel_tooltip(true); - return true; + return IPython.tooltip.remove_and_cancel_tooltip(true); } else if (event.which === key.DOWNARROW && event.type === 'keydown') { // If we are not at the bottom, let CM handle the down arrow and // prevent the global keydown handler from handling it. @@ -242,9 +241,12 @@ var IPython = (function (IPython) { * @method execute */ CodeCell.prototype.execute = function () { - this.output_area.clear_output(true, true, true); + this.output_area.clear_output(); this.set_input_prompt('*'); this.element.addClass("running"); + if (this.last_msg_id) { + this.kernel.clear_callbacks_for_msg(this.last_msg_id); + } var callbacks = { 'execute_reply': $.proxy(this._handle_execute_reply, this), 'output': $.proxy(this.output_area.handle_output, this.output_area), @@ -252,7 +254,7 @@ var IPython = (function (IPython) { 'set_next_input': $.proxy(this._handle_set_next_input, this), 'input_request': $.proxy(this._handle_input_request, this) }; - var msg_id = this.kernel.execute(this.get_text(), callbacks, {silent: false, store_history: true}); + this.last_msg_id = this.kernel.execute(this.get_text(), callbacks, {silent: false, store_history: true}); }; /** @@ -273,7 +275,7 @@ var IPython = (function (IPython) { var data = {'cell': this, 'text': text} $([IPython.events]).trigger('set_next_input.Notebook', data); } - + /** * @method _handle_input_request * @private @@ -388,8 +390,8 @@ var IPython = (function (IPython) { }; - CodeCell.prototype.clear_output = function (stdout, stderr, other) { - this.output_area.clear_output(stdout, stderr, other); + CodeCell.prototype.clear_output = function (wait) { + this.output_area.clear_output(wait); }; diff --git a/IPython/html/static/notebook/js/config.js b/IPython/html/static/notebook/js/config.js index afb0eb7..fc5bc7c 100644 --- a/IPython/html/static/notebook/js/config.js +++ b/IPython/html/static/notebook/js/config.js @@ -50,12 +50,13 @@ var IPython = (function (IPython) { * cell_magic_highlight['javascript'] = {'reg':[/^var/]} */ cell_magic_highlight : { - 'magic_javascript':{'reg':[/^%%javascript/]} - ,'magic_perl' :{'reg':[/^%%perl/]} - ,'magic_ruby' :{'reg':[/^%%ruby/]} - ,'magic_python' :{'reg':[/^%%python3?/]} - ,'magic_shell' :{'reg':[/^%%bash/]} - ,'magic_r' :{'reg':[/^%%R/]} + 'magic_javascript' :{'reg':[/^%%javascript/]} + ,'magic_perl' :{'reg':[/^%%perl/]} + ,'magic_ruby' :{'reg':[/^%%ruby/]} + ,'magic_python' :{'reg':[/^%%python3?/]} + ,'magic_shell' :{'reg':[/^%%bash/]} + ,'magic_r' :{'reg':[/^%%R/]} + ,'magic_text/x-cython' :{'reg':[/^%%cython/]} }, /** diff --git a/IPython/html/static/notebook/js/main.js b/IPython/html/static/notebook/js/main.js index 4f3b72b..ec25f23 100644 --- a/IPython/html/static/notebook/js/main.js +++ b/IPython/html/static/notebook/js/main.js @@ -99,6 +99,10 @@ function (marked) { tables: true, langPrefix: "language-", highlight: function(code, lang) { + if (!lang) { + // no language, no highlight + return code; + } var highlighted; try { highlighted = hljs.highlight(lang, code, false); diff --git a/IPython/html/static/notebook/js/notebook.js b/IPython/html/static/notebook/js/notebook.js index f5df0dc..88891ac 100644 --- a/IPython/html/static/notebook/js/notebook.js +++ b/IPython/html/static/notebook/js/notebook.js @@ -1175,8 +1175,14 @@ var IPython = (function (IPython) { Notebook.prototype.merge_cell_above = function () { var index = this.get_selected_index(); var cell = this.get_cell(index); + if (!cell.is_mergeable()) { + return; + } if (index > 0) { var upper_cell = this.get_cell(index-1); + if (!upper_cell.is_mergeable()) { + return; + } var upper_text = upper_cell.get_text(); var text = cell.get_text(); if (cell instanceof IPython.CodeCell) { @@ -1199,8 +1205,14 @@ var IPython = (function (IPython) { Notebook.prototype.merge_cell_below = function () { var index = this.get_selected_index(); var cell = this.get_cell(index); + if (!cell.is_mergeable()) { + return; + } if (index < this.ncells()-1) { var lower_cell = this.get_cell(index+1); + if (!lower_cell.is_mergeable()) { + return; + } var lower_text = lower_cell.get_text(); var text = cell.get_text(); if (cell instanceof IPython.CodeCell) { @@ -1327,7 +1339,7 @@ var IPython = (function (IPython) { var cells = this.get_cells(); for (var i=0; i').html(msg + "
" + + err.toString() + + '
See your browser Javascript console for more details.' + ).addClass('js-error') + ); + container.show(); + }; + + OutputArea.prototype._safe_append = function (toinsert) { + // safely append an item to the document + // this is an object created by user code, + // and may have errors, which should not be raised + // under any circumstances. + try { + this.element.append(toinsert); + } catch(err) { + console.log(err); + this._append_javascript_error(err, this.element); + } + }; OutputArea.prototype.append_pyout = function (json, dynamic) { @@ -321,19 +354,7 @@ var IPython = (function (IPython) { toinsert.find('div.prompt').addClass('output_prompt').html('Out[' + n + ']:'); } this.append_mime_type(json, toinsert, dynamic); - try { - this.element.append(toinsert); - } catch(err) { - console.log("Error attaching output!"); - console.log(err); - this.element.show(); - toinsert.html($('

') - .html("Javascript error adding output!
" + - err.toString() + - '
See your browser Javascript console for more details.') - .addClass('js-error') - ); - } + this._safe_append(toinsert); // If we just output latex, typeset it. if ((json.latex !== undefined) || (json.html !== undefined)) { this.typeset(); @@ -352,7 +373,7 @@ var IPython = (function (IPython) { s = s + '\n'; var toinsert = this.create_output_area(); this.append_text(s, {}, toinsert); - this.element.append(toinsert); + this._safe_append(toinsert); } }; @@ -389,14 +410,14 @@ var IPython = (function (IPython) { // If we got here, attach a new div var toinsert = this.create_output_area(); this.append_text(text, {}, toinsert, "output_stream "+subclass); - this.element.append(toinsert); + this._safe_append(toinsert); }; OutputArea.prototype.append_display_data = function (json, dynamic) { var toinsert = this.create_output_area(); this.append_mime_type(json, toinsert, dynamic); - this.element.append(toinsert); + this._safe_append(toinsert); // If we just output latex, typeset it. if ( (json.latex !== undefined) || (json.html !== undefined) ) { this.typeset(); @@ -444,15 +465,7 @@ var IPython = (function (IPython) { try { eval(js); } catch(err) { - console.log('Error in Javascript!'); - console.log(err); - container.show(); - element.append($('
') - .html("Error in Javascript !
"+ - err.toString()+ - '
See your browser Javascript console for more details.') - .addClass('js-error') - ); + this._append_javascript_error(err, container); } }; @@ -546,7 +559,6 @@ var IPython = (function (IPython) { OutputArea.prototype.append_raw_input = function (content) { var that = this; this.expand(); - this.flush_clear_timeout(); var area = this.create_output_area(); // disable any other raw_inputs, if they are left around @@ -599,81 +611,35 @@ var IPython = (function (IPython) { OutputArea.prototype.handle_clear_output = function (content) { - this.clear_output(content.stdout, content.stderr, content.other); + this.clear_output(content.wait); }; - OutputArea.prototype.clear_output = function (stdout, stderr, other) { - var that = this; - if (this.clear_out_timeout != null){ - // fire previous pending clear *immediately* - clearTimeout(this.clear_out_timeout); - this.clear_out_timeout = null; - this.clear_output_callback(this._clear_stdout, this._clear_stderr, this._clear_other); - } - // store flags for flushing the timeout - this._clear_stdout = stdout; - this._clear_stderr = stderr; - this._clear_other = other; - this.clear_out_timeout = setTimeout(function() { - // really clear timeout only after a short delay - // this reduces flicker in 'clear_output; print' cases - that.clear_out_timeout = null; - that._clear_stdout = that._clear_stderr = that._clear_other = null; - that.clear_output_callback(stdout, stderr, other); - }, 500 - ); - }; + OutputArea.prototype.clear_output = function(wait) { + if (wait) { + // If a clear is queued, clear before adding another to the queue. + if (this.clear_queued) { + this.clear_output(false); + }; - OutputArea.prototype.clear_output_callback = function (stdout, stderr, other) { - var output_div = this.element; + this.clear_queued = true; + } else { - if (stdout && stderr && other){ + // Fix the output div's height if the clear_output is waiting for + // new output (it is being used in an animation). + if (this.clear_queued) { + var height = this.element.height(); + this.element.height(height); + this.clear_queued = false; + } + // clear all, no need for logic - output_div.html(""); + this.element.html(""); this.outputs = []; this.unscroll_area(); return; - } - // remove html output - // each output_subarea that has an identifying class is in an output_area - // which is the element to be removed. - if (stdout) { - output_div.find("div.output_stdout").parent().remove(); - } - if (stderr) { - output_div.find("div.output_stderr").parent().remove(); - } - if (other) { - output_div.find("div.output_subarea").not("div.output_stderr").not("div.output_stdout").parent().remove(); - } - this.unscroll_area(); - - // remove cleared outputs from JSON list: - for (var i = this.outputs.length - 1; i >= 0; i--) { - var out = this.outputs[i]; - var output_type = out.output_type; - if (output_type == "display_data" && other) { - this.outputs.splice(i,1); - } else if (output_type == "stream") { - if (stdout && out.stream == "stdout") { - this.outputs.splice(i,1); - } else if (stderr && out.stream == "stderr") { - this.outputs.splice(i,1); - } - } - } - }; - - - OutputArea.prototype.flush_clear_timeout = function() { - var output_div = this.element; - if (this.clear_out_timeout){ - clearTimeout(this.clear_out_timeout); - this.clear_out_timeout = null; - this.clear_output_callback(this._clear_stdout, this._clear_stderr, this._clear_other); - } + }; }; diff --git a/IPython/html/static/notebook/js/savewidget.js b/IPython/html/static/notebook/js/savewidget.js index 8e2a16c..6fee780 100644 --- a/IPython/html/static/notebook/js/savewidget.js +++ b/IPython/html/static/notebook/js/savewidget.js @@ -52,7 +52,7 @@ var IPython = (function (IPython) { $([IPython.events]).on('checkpoints_listed.Notebook', function (event, data) { that.set_last_checkpoint(data[0]); }); - + $([IPython.events]).on('checkpoint_created.Notebook', function (event, data) { that.set_last_checkpoint(data); }); @@ -104,7 +104,7 @@ var IPython = (function (IPython) { return false; } }); - that.find('input[type="text"]').focus(); + that.find('input[type="text"]').focus().select(); } }); } diff --git a/IPython/html/static/notebook/js/textcell.js b/IPython/html/static/notebook/js/textcell.js index d88144d..7e63591 100644 --- a/IPython/html/static/notebook/js/textcell.js +++ b/IPython/html/static/notebook/js/textcell.js @@ -482,6 +482,22 @@ var IPython = (function (IPython) { return data; }; + /** + * can the cell be split into two cells + * @method is_splittable + **/ + HeadingCell.prototype.is_splittable = function () { + return false; + }; + + + /** + * can the cell be merged with other cells + * @method is_mergeable + **/ + HeadingCell.prototype.is_mergeable = function () { + return false; + }; /** * Change heading level of cell, and re-render diff --git a/IPython/html/static/notebook/js/tooltip.js b/IPython/html/static/notebook/js/tooltip.js index 4d324fe..66db57e 100644 --- a/IPython/html/static/notebook/js/tooltip.js +++ b/IPython/html/static/notebook/js/tooltip.js @@ -161,16 +161,23 @@ var IPython = (function (IPython) { this.code_mirror = null; } + // return true on successfully removing a visible tooltip; otherwise return + // false. Tooltip.prototype.remove_and_cancel_tooltip = function (force) { // note that we don't handle closing directly inside the calltip // as in the completer, because it is not focusable, so won't // get the event. - if (this._sticky == false || force == true) { - this.cancel_stick(); - this._hide(); + if (!this._hidden) { + if (force || !this._sticky) { + this.cancel_stick(); + this._hide(); + } + this.cancel_pending(); + this.reset_tabs_function(); + return true; + } else { + return false; } - this.cancel_pending(); - this.reset_tabs_function(); } // cancel autocall done after '(' for example. @@ -335,7 +342,7 @@ var IPython = (function (IPython) { if (docstring == null) { docstring = reply.docstring; } - + if (docstring == null) { // For reals this time, no docstring if (this._hide_if_no_docstring) { diff --git a/IPython/html/static/notebook/less/cell.less b/IPython/html/static/notebook/less/cell.less index 851fcde..30d6719 100644 --- a/IPython/html/static/notebook/less/cell.less +++ b/IPython/html/static/notebook/less/cell.less @@ -1,4 +1,4 @@ -.cell { +div.cell { border: 1px solid transparent; .vbox(); @@ -6,9 +6,6 @@ .corner-all; border : thin @border_color solid; } -} - -div.cell { width: 100%; padding: 5px 5px 5px 0px; /* This acts as a spacer between cells, that is outside the border */ diff --git a/IPython/html/static/notebook/less/codemirror.less b/IPython/html/static/notebook/less/codemirror.less index f3a257c..0de34f4 100644 --- a/IPython/html/static/notebook/less/codemirror.less +++ b/IPython/html/static/notebook/less/codemirror.less @@ -22,7 +22,7 @@ overflow-x: auto; } -@-moz-document { +@-moz-document url-prefix() { /* Firefox does weird and terrible things (#3549) when overflow-x is auto */ /* It doesn't respect the overflow setting anyway, so we can workaround it with this */ .CodeMirror-scroll { diff --git a/IPython/html/static/services/kernels/js/kernel.js b/IPython/html/static/services/kernels/js/kernel.js index 90b888e..b3c2e83 100644 --- a/IPython/html/static/services/kernels/js/kernel.js +++ b/IPython/html/static/services/kernels/js/kernel.js @@ -119,7 +119,6 @@ var IPython = (function (IPython) { this.ws_url = ws_url; this.kernel_url = this.base_url + "/" + this.kernel_id; this.start_channels(); - $([IPython.events]).trigger('status_started.Kernel', {kernel: this}); }; @@ -144,11 +143,7 @@ var IPython = (function (IPython) { this.shell_channel = new this.WebSocket(ws_url + "/shell"); this.stdin_channel = new this.WebSocket(ws_url + "/stdin"); this.iopub_channel = new this.WebSocket(ws_url + "/iopub"); - send_cookie = function(){ - // send the session id so the Session object Python-side - // has the same identity - this.send(that.session_id + ':' + document.cookie); - }; + var already_called_onclose = false; // only alert once var ws_closed_early = function(evt){ if (already_called_onclose){ @@ -170,7 +165,7 @@ var IPython = (function (IPython) { }; var channels = [this.shell_channel, this.iopub_channel, this.stdin_channel]; for (var i=0; i < channels.length; i++) { - channels[i].onopen = send_cookie; + channels[i].onopen = $.proxy(this._ws_opened, this); channels[i].onclose = ws_closed_early; } // switch from early-close to late-close message after 1s @@ -187,7 +182,27 @@ var IPython = (function (IPython) { }; /** - * Stop the `shell`and `iopub` channels. + * Handle a websocket entering the open state + * sends session and cookie authentication info as first message. + * Once all sockets are open, signal the Kernel.status_started event. + * @method _ws_opened + */ + Kernel.prototype._ws_opened = function (evt) { + // send the session id so the Session object Python-side + // has the same identity + evt.target.send(this.session_id + ':' + document.cookie); + + var channels = [this.shell_channel, this.iopub_channel, this.stdin_channel]; + for (var i=0; i < channels.length; i++) { + // if any channel is not ready, don't trigger event. + if ( !channels[i].readyState ) return; + } + // all events ready, trigger started event. + $([IPython.events]).trigger('status_started.Kernel', {kernel: this}); + }; + + /** + * Stop the websocket channels. * @method stop_channels */ Kernel.prototype.stop_channels = function () { @@ -388,9 +403,16 @@ var IPython = (function (IPython) { }; + Kernel.prototype.clear_callbacks_for_msg = function (msg_id) { + if (this._msg_callbacks[msg_id] !== undefined ) { + delete this._msg_callbacks[msg_id]; + } + }; + + Kernel.prototype.set_callbacks_for_msg = function (msg_id, callbacks) { this._msg_callbacks[msg_id] = callbacks || {}; - } + }; Kernel.prototype._handle_shell_reply = function (e) { diff --git a/IPython/html/static/style/ipython.min.css b/IPython/html/static/style/ipython.min.css index 8d8fbe3..df2a7b8 100644 --- a/IPython/html/static/style/ipython.min.css +++ b/IPython/html/static/style/ipython.min.css @@ -18,20 +18,6 @@ .start{-webkit-box-pack:start;-moz-box-pack:start;box-pack:start;} .end{-webkit-box-pack:end;-moz-box-pack:end;box-pack:end;} .center{-webkit-box-pack:center;-moz-box-pack:center;box-pack:center;} -body{background-color:white;position:absolute;left:0px;right:0px;top:0px;bottom:0px;overflow:visible;} -div#header{display:none;} -#ipython_notebook{padding-left:16px;} -#noscript{width:auto;padding-top:16px;padding-bottom:16px;text-align:center;font-size:22px;color:red;font-weight:bold;} -#ipython_notebook img{font-family:Verdana,"Helvetica Neue",Arial,Helvetica,Geneva,sans-serif;height:24px;text-decoration:none;color:black;} -#site{width:100%;display:none;} -.ui-button .ui-button-text{padding:0.2em 0.8em;font-size:77%;} -input.ui-button{padding:0.3em 0.9em;} -.navbar span{margin-top:3px;} -span#login_widget{float:right;} -.nav-header{text-transform:none;} -.navbar-nobg{background-color:transparent;background-image:none;} -#header>span{margin-top:10px;} -.modal-body{max-height:500px;} .center-nav{display:inline-block;margin-bottom:-4px;} .alternate_upload{background-color:none;display:inline;} .alternate_upload.form{padding:0;margin:0;} @@ -67,8 +53,7 @@ input.engine_num_input{height:20px;margin-bottom:2px;padding-top:0;padding-botto .ansibgpurple{background-color:magenta;} .ansibgcyan{background-color:cyan;} .ansibggray{background-color:gray;} -.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;}.cell.selected{border-radius:4px;border:thin #ababab solid;} -div.cell{width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;} +div.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;}div.cell.selected{border-radius:4px;border:thin #ababab solid;} div.prompt{width:11ex;padding:0.4em;margin:0px;font-family:monospace;text-align:right;line-height:1.231em;} .celltoolbar{border:thin solid #CFCFCF;border-bottom:none;background:#EEE;border-top-right-radius:3px;border-top-left-radius:3px;width:100%;-webkit-box-pack:end;height:22px;} .no_input_radius{border-top-right-radius:0px;border-top-left-radius:0px;} @@ -98,7 +83,7 @@ div.out_prompt_overlay:hover{-webkit-box-shadow:inset 0 0 1px #000000;-moz-box-s div.output_prompt{color:darkred;} .CodeMirror{line-height:1.231em;height:auto;background:none;} .CodeMirror-scroll{overflow-y:hidden;overflow-x:auto;} -@-moz-document {.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;} +@-moz-document url-prefix(){.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;} .CodeMirror-linenumber{padding:0 8px 0 4px;} .CodeMirror-gutters{border-bottom-left-radius:4px;border-top-left-radius:4px;} .CodeMirror pre{padding:0;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} diff --git a/IPython/html/static/style/style.less b/IPython/html/static/style/style.less index fc751b1..a0eb839 100644 --- a/IPython/html/static/style/style.less +++ b/IPython/html/static/style/style.less @@ -8,6 +8,7 @@ // base @import "../base/less/style.less"; +@import "../base/less/page.less"; // auth @import "../auth/less/style.less"; diff --git a/IPython/html/static/style/style.min.css b/IPython/html/static/style/style.min.css index c87d467..b14ceca 100644 --- a/IPython/html/static/style/style.min.css +++ b/IPython/html/static/style/style.min.css @@ -1434,8 +1434,7 @@ input.engine_num_input{height:20px;margin-bottom:2px;padding-top:0;padding-botto .ansibgpurple{background-color:magenta;} .ansibgcyan{background-color:cyan;} .ansibggray{background-color:gray;} -.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;}.cell.selected{border-radius:4px;border:thin #ababab solid;} -div.cell{width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;} +div.cell{border:1px solid transparent;display:-webkit-box;-webkit-box-orient:vertical;-webkit-box-align:stretch;display:-moz-box;-moz-box-orient:vertical;-moz-box-align:stretch;display:box;box-orient:vertical;box-align:stretch;width:100%;padding:5px 5px 5px 0px;margin:2px 0px 2px 7px;outline:none;}div.cell.selected{border-radius:4px;border:thin #ababab solid;} div.prompt{width:11ex;padding:0.4em;margin:0px;font-family:monospace;text-align:right;line-height:1.231em;} .celltoolbar{border:thin solid #CFCFCF;border-bottom:none;background:#EEE;border-top-right-radius:3px;border-top-left-radius:3px;width:100%;-webkit-box-pack:end;height:22px;} .no_input_radius{border-top-right-radius:0px;border-top-left-radius:0px;} @@ -1465,7 +1464,7 @@ div.out_prompt_overlay:hover{-webkit-box-shadow:inset 0 0 1px #000000;-moz-box-s div.output_prompt{color:darkred;} .CodeMirror{line-height:1.231em;height:auto;background:none;} .CodeMirror-scroll{overflow-y:hidden;overflow-x:auto;} -@-moz-document {.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;} +@-moz-document url-prefix(){.CodeMirror-scroll{overflow-x:hidden;}}.CodeMirror-lines{padding:0.4em;} .CodeMirror-linenumber{padding:0 8px 0 4px;} .CodeMirror-gutters{border-bottom-left-radius:4px;border-top-left-radius:4px;} .CodeMirror pre{padding:0;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;} diff --git a/IPython/html/tests/test_notebookapp.py b/IPython/html/tests/test_notebookapp.py new file mode 100644 index 0000000..56509cd --- /dev/null +++ b/IPython/html/tests/test_notebookapp.py @@ -0,0 +1,25 @@ +"""Test NotebookApp""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2013 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import nose.tools as nt + +import IPython.testing.tools as tt + +#----------------------------------------------------------------------------- +# Test functions +#----------------------------------------------------------------------------- + +def test_help_output(): + """ipython notebook --help-all works""" + tt.help_all_output_test('notebook') + diff --git a/IPython/kernel/connect.py b/IPython/kernel/connect.py index 06db190..8ce939d 100644 --- a/IPython/kernel/connect.py +++ b/IPython/kernel/connect.py @@ -34,14 +34,13 @@ import zmq from IPython.external.ssh import tunnel # IPython imports -# from IPython.config import Configurable +from IPython.config import Configurable from IPython.core.profiledir import ProfileDir -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.path import filefind, get_ipython_dir from IPython.utils.py3compat import str_to_bytes, bytes_to_str from IPython.utils.traitlets import ( Bool, Integer, Unicode, CaselessStrEnum, - HasTraits, ) @@ -50,7 +49,7 @@ from IPython.utils.traitlets import ( #----------------------------------------------------------------------------- def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0, - control_port=0, ip=LOCALHOST, key=b'', transport='tcp', + control_port=0, ip='', key=b'', transport='tcp', signature_scheme='hmac-sha256', ): """Generates a JSON config file, including the selection of random ports. @@ -91,6 +90,8 @@ def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, and 'sha256' is the default hash function. """ + if not ip: + ip = localhost() # default to temporary connector file if not fname: fname = tempfile.mktemp('.json') @@ -383,7 +384,7 @@ channel_socket_types = { port_names = [ "%s_port" % channel for channel in ('shell', 'stdin', 'iopub', 'hb', 'control')] -class ConnectionFileMixin(HasTraits): +class ConnectionFileMixin(Configurable): """Mixin for configurable classes that work with connection files""" # The addresses for the communication channels @@ -392,7 +393,7 @@ class ConnectionFileMixin(HasTraits): transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True) - ip = Unicode(LOCALHOST, config=True, + ip = Unicode(config=True, help="""Set the kernel\'s IP address [default localhost]. If the IP address is something other than localhost, then Consoles on other machines will be able to connect @@ -406,7 +407,7 @@ class ConnectionFileMixin(HasTraits): else: return 'kernel-ipc' else: - return LOCALHOST + return localhost() def _ip_changed(self, name, old, new): if new == '*': diff --git a/IPython/kernel/manager.py b/IPython/kernel/manager.py index cf5efe1..a26423e 100644 --- a/IPython/kernel/manager.py +++ b/IPython/kernel/manager.py @@ -1,5 +1,4 @@ -"""Base class to manage a running kernel -""" +"""Base class to manage a running kernel""" #----------------------------------------------------------------------------- # Copyright (C) 2013 The IPython Development Team @@ -24,7 +23,7 @@ import zmq # Local imports from IPython.config.configurable import LoggingConfigurable from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCAL_IPS +from IPython.utils.localinterfaces import is_local_ip, local_ips from IPython.utils.traitlets import ( Any, Instance, Unicode, List, Bool, Type, DottedObjectName ) @@ -185,11 +184,11 @@ class KernelManager(LoggingConfigurable, ConnectionFileMixin): keyword arguments that are passed down to build the kernel_cmd and launching the kernel (e.g. Popen kwargs). """ - if self.transport == 'tcp' and self.ip not in LOCAL_IPS: + if self.transport == 'tcp' and not is_local_ip(self.ip): raise RuntimeError("Can only launch a kernel on a local interface. " "Make sure that the '*_address' attributes are " "configured properly. " - "Currently valid addresses are: %s"%LOCAL_IPS + "Currently valid addresses are: %s" % local_ips() ) # write connection file / get default ports diff --git a/IPython/kernel/multikernelmanager.py b/IPython/kernel/multikernelmanager.py index e3c2d41..a665573 100644 --- a/IPython/kernel/multikernelmanager.py +++ b/IPython/kernel/multikernelmanager.py @@ -171,7 +171,7 @@ class MultiKernelManager(LoggingConfigurable): self.log.info("Signaled Kernel %s with %s" % (kernel_id, signum)) @kernel_method - def restart_kernel(self, kernel_id): + def restart_kernel(self, kernel_id, now=False): """Restart a kernel by its uuid, keeping the same ports. Parameters diff --git a/IPython/kernel/tests/test_kernel.py b/IPython/kernel/tests/test_kernel.py index 305dce7..0e834c4 100644 --- a/IPython/kernel/tests/test_kernel.py +++ b/IPython/kernel/tests/test_kernel.py @@ -11,101 +11,18 @@ # Imports #------------------------------------------------------------------------------- -import os -import shutil import sys -import tempfile - -from contextlib import contextmanager -from subprocess import PIPE import nose.tools as nt -from IPython.kernel import KernelManager -from IPython.kernel.tests.test_message_spec import execute, flush_channels -from IPython.testing import decorators as dec -from IPython.utils import path +from IPython.testing import decorators as dec, tools as tt +from IPython.utils import py3compat + +from .utils import new_kernel, kernel, TIMEOUT, assemble_output, execute, flush_channels #------------------------------------------------------------------------------- # Tests #------------------------------------------------------------------------------- -IPYTHONDIR = None -save_env = None -save_get_ipython_dir = None - -def setup(): - """setup temporary IPYTHONDIR for tests""" - global IPYTHONDIR - global save_env - global save_get_ipython_dir - - IPYTHONDIR = tempfile.mkdtemp() - - save_env = os.environ.copy() - os.environ["IPYTHONDIR"] = IPYTHONDIR - - save_get_ipython_dir = path.get_ipython_dir - path.get_ipython_dir = lambda : IPYTHONDIR - - -def teardown(): - path.get_ipython_dir = save_get_ipython_dir - os.environ = save_env - - try: - shutil.rmtree(IPYTHONDIR) - except (OSError, IOError): - # no such file - pass - - -@contextmanager -def new_kernel(): - """start a kernel in a subprocess, and wait for it to be ready - - Returns - ------- - kernel_manager: connected KernelManager instance - """ - KM = KernelManager() - - KM.start_kernel(stdout=PIPE, stderr=PIPE) - KC = KM.client() - KC.start_channels() - - # wait for kernel to be ready - KC.shell_channel.execute("import sys") - KC.shell_channel.get_msg(block=True, timeout=5) - flush_channels(KC) - try: - yield KC - finally: - KC.stop_channels() - KM.shutdown_kernel() - - -def assemble_output(iopub): - """assemble stdout/err from an execution""" - stdout = '' - stderr = '' - while True: - msg = iopub.get_msg(block=True, timeout=1) - msg_type = msg['msg_type'] - content = msg['content'] - if msg_type == 'status' and content['execution_state'] == 'idle': - # idle message signals end of output - break - elif msg['msg_type'] == 'stream': - if content['name'] == 'stdout': - stdout = stdout + content['data'] - elif content['name'] == 'stderr': - stderr = stderr + content['data'] - else: - raise KeyError("bad stream: %r" % content['name']) - else: - # other output, ignored - pass - return stdout, stderr def _check_mp_mode(kc, expected=False, stream="stdout"): @@ -116,16 +33,17 @@ def _check_mp_mode(kc, expected=False, stream="stdout"): nt.assert_equal(eval(stdout.strip()), expected) +# printing tests + def test_simple_print(): """simple print statement in kernel""" - with new_kernel() as kc: + with kernel() as kc: iopub = kc.iopub_channel msg_id, content = execute(kc=kc, code="print ('hi')") stdout, stderr = assemble_output(iopub) nt.assert_equal(stdout, 'hi\n') nt.assert_equal(stderr, '') _check_mp_mode(kc, expected=False) - print ('hello') @dec.knownfailureif(sys.platform == 'win32', "subprocess prints fail on Windows") @@ -161,7 +79,7 @@ def test_subprocess_print(): def test_subprocess_noprint(): """mp.Process without print doesn't trigger iostream mp_mode""" - with new_kernel() as kc: + with kernel() as kc: iopub = kc.iopub_channel np = 5 @@ -202,3 +120,51 @@ def test_subprocess_error(): _check_mp_mode(kc, expected=False) _check_mp_mode(kc, expected=False, stream="stderr") +# raw_input tests + +def test_raw_input(): + """test [raw_]input""" + with kernel() as kc: + iopub = kc.iopub_channel + + input_f = "input" if py3compat.PY3 else "raw_input" + theprompt = "prompt> " + code = 'print({input_f}("{theprompt}"))'.format(**locals()) + msg_id = kc.execute(code, allow_stdin=True) + msg = kc.get_stdin_msg(block=True, timeout=TIMEOUT) + nt.assert_equal(msg['header']['msg_type'], u'input_request') + content = msg['content'] + nt.assert_equal(content['prompt'], theprompt) + text = "some text" + kc.input(text) + reply = kc.get_shell_msg(block=True, timeout=TIMEOUT) + nt.assert_equal(reply['content']['status'], 'ok') + stdout, stderr = assemble_output(iopub) + nt.assert_equal(stdout, text + "\n") + + +@dec.skipif(py3compat.PY3) +def test_eval_input(): + """test input() on Python 2""" + with kernel() as kc: + iopub = kc.iopub_channel + + input_f = "input" if py3compat.PY3 else "raw_input" + theprompt = "prompt> " + code = 'print(input("{theprompt}"))'.format(**locals()) + msg_id = kc.execute(code, allow_stdin=True) + msg = kc.get_stdin_msg(block=True, timeout=TIMEOUT) + nt.assert_equal(msg['header']['msg_type'], u'input_request') + content = msg['content'] + nt.assert_equal(content['prompt'], theprompt) + kc.input("1+1") + reply = kc.get_shell_msg(block=True, timeout=TIMEOUT) + nt.assert_equal(reply['content']['status'], 'ok') + stdout, stderr = assemble_output(iopub) + nt.assert_equal(stdout, "2\n") + + +def test_help_output(): + """ipython kernel --help-all works""" + tt.help_all_output_test('kernel') + diff --git a/IPython/kernel/tests/test_kernelmanager.py b/IPython/kernel/tests/test_kernelmanager.py index ad6fef4..fdefe2d 100644 --- a/IPython/kernel/tests/test_kernelmanager.py +++ b/IPython/kernel/tests/test_kernelmanager.py @@ -26,18 +26,11 @@ class TestKernelManager(TestCase): def _run_lifecycle(self, km): km.start_kernel(stdout=PIPE, stderr=PIPE) self.assertTrue(km.is_alive()) - km.restart_kernel() + km.restart_kernel(now=True) self.assertTrue(km.is_alive()) - # We need a delay here to give the restarting kernel a chance to - # restart. Otherwise, the interrupt will kill it, causing the test - # suite to hang. The reason it *hangs* is that the shutdown - # message for the restart sometimes hasn't been sent to the kernel. - # Because linger is oo on the shell channel, the context can't - # close until the message is sent to the kernel, which is not dead. - time.sleep(1.0) km.interrupt_kernel() self.assertTrue(isinstance(km, KernelManager)) - km.shutdown_kernel() + km.shutdown_kernel(now=True) def test_tcp_lifecycle(self): km = self._get_tcp_km() diff --git a/IPython/kernel/tests/test_launcher.py b/IPython/kernel/tests/test_launcher.py index 5aa069e..0936790 100644 --- a/IPython/kernel/tests/test_launcher.py +++ b/IPython/kernel/tests/test_launcher.py @@ -20,14 +20,12 @@ Authors import nose.tools as nt # Our own imports -from IPython.testing import decorators as dec from IPython.kernel.launcher import swallow_argv #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- -@dec.parametric def test_swallow_argv(): tests = [ # expected , argv , aliases, flags @@ -56,5 +54,5 @@ def test_swallow_argv(): "expected : %r" % expected, "returned : %r" % stripped, ]) - yield nt.assert_equal(expected, stripped, message) + nt.assert_equal(expected, stripped, message) diff --git a/IPython/kernel/tests/test_message_spec.py b/IPython/kernel/tests/test_message_spec.py index f73cfe8..29c56cb 100644 --- a/IPython/kernel/tests/test_message_spec.py +++ b/IPython/kernel/tests/test_message_spec.py @@ -1,7 +1,7 @@ -"""Test suite for our zeromq-based messaging specification. +"""Test suite for our zeromq-based message specification. """ #----------------------------------------------------------------------------- -# Copyright (C) 2010-2011 The IPython Development Team +# Copyright (C) 2010 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING.txt, distributed as part of this software. @@ -15,77 +15,25 @@ import nose.tools as nt from IPython.kernel import KernelManager -from IPython.testing import decorators as dec from IPython.utils.traitlets import ( HasTraits, TraitError, Bool, Unicode, Dict, Integer, List, Enum, Any, ) +from .utils import TIMEOUT, start_global_kernel, flush_channels, execute + #----------------------------------------------------------------------------- -# Global setup and utilities +# Globals #----------------------------------------------------------------------------- - -STARTUP_TIMEOUT = 60 -TIMEOUT = 15 +KC = None def setup(): - global KM, KC - KM = KernelManager() - KM.start_kernel(stdout=PIPE, stderr=PIPE) - KC = KM.client() - KC.start_channels() - - # wait for kernel to be ready - try: - msg = KC.iopub_channel.get_msg(block=True, timeout=STARTUP_TIMEOUT) - except Empty: - pass - msg_id = KC.kernel_info() - KC.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT) - flush_channels() - - -def teardown(): - KC.stop_channels() - KM.shutdown_kernel() - - -def flush_channels(kc=None): - """flush any messages waiting on the queue""" - if kc is None: - kc = KC - for channel in (kc.shell_channel, kc.iopub_channel): - while True: - try: - msg = channel.get_msg(block=True, timeout=0.1) - except Empty: - break - else: - list(validate_message(msg)) - - -def execute(code='', kc=None, **kwargs): - """wrapper for doing common steps for validating an execution request""" - if kc is None: - kc = KC - msg_id = kc.execute(code=code, **kwargs) - reply = kc.get_shell_msg(timeout=TIMEOUT) - list(validate_message(reply, 'execute_reply', msg_id)) - busy = kc.get_iopub_msg(timeout=TIMEOUT) - list(validate_message(busy, 'status', msg_id)) - nt.assert_equal(busy['content']['execution_state'], 'busy') - - if not kwargs.get('silent'): - pyin = kc.get_iopub_msg(timeout=TIMEOUT) - list(validate_message(pyin, 'pyin', msg_id)) - nt.assert_equal(pyin['content']['code'], code) - - return msg_id, reply['content'] + global KC + KC = start_global_kernel() #----------------------------------------------------------------------------- -# MSG Spec References +# Message Spec References #----------------------------------------------------------------------------- - class Reference(HasTraits): """ @@ -101,14 +49,14 @@ class Reference(HasTraits): def check(self, d): """validate a dict against our traits""" for key in self.trait_names(): - yield nt.assert_true(key in d, "Missing key: %r, should be found in %s" % (key, d)) + nt.assert_in(key, d) # FIXME: always allow None, probably not a good idea if d[key] is None: continue try: setattr(self, key, d[key]) except TraitError as e: - yield nt.assert_true(False, str(e)) + nt.assert_true(False, str(e)) class RMessage(Reference): @@ -133,14 +81,11 @@ class ExecuteReply(Reference): status = Enum((u'ok', u'error')) def check(self, d): - for tst in Reference.check(self, d): - yield tst + Reference.check(self, d) if d['status'] == 'ok': - for tst in ExecuteReplyOkay().check(d): - yield tst + ExecuteReplyOkay().check(d) elif d['status'] == 'error': - for tst in ExecuteReplyError().check(d): - yield tst + ExecuteReplyError().check(d) class ExecuteReplyOkay(Reference): @@ -177,11 +122,9 @@ class OInfoReply(Reference): source = Unicode() def check(self, d): - for tst in Reference.check(self, d): - yield tst + Reference.check(self, d) if d['argspec'] is not None: - for tst in ArgSpec().check(d['argspec']): - yield tst + ArgSpec().check(d['argspec']) class ArgSpec(Reference): @@ -212,10 +155,8 @@ class KernelInfoReply(Reference): def _ipython_version_changed(self, name, old, new): for v in new: - nt.assert_true( - isinstance(v, int) or isinstance(v, basestring), - 'expected int or string as version component, got {0!r}' - .format(v)) + assert isinstance(v, int) or isinstance(v, basestring), \ + 'expected int or string as version component, got {0!r}'.format(v) # IOPub messages @@ -241,8 +182,8 @@ class DisplayData(Reference): data = Dict() def _data_changed(self, name, old, new): for k,v in new.iteritems(): - nt.assert_true(mime_pat.match(k)) - nt.assert_true(isinstance(v, basestring), "expected string data, got %r" % v) + assert mime_pat.match(k) + nt.assert_is_instance(v, basestring) class PyOut(Reference): @@ -250,8 +191,8 @@ class PyOut(Reference): data = Dict() def _data_changed(self, name, old, new): for k,v in new.iteritems(): - nt.assert_true(mime_pat.match(k)) - nt.assert_true(isinstance(v, basestring), "expected string data, got %r" % v) + assert mime_pat.match(k) + nt.assert_is_instance(v, basestring) references = { @@ -282,13 +223,12 @@ def validate_message(msg, msg_type=None, parent=None): """ RMessage().check(msg) if msg_type: - yield nt.assert_equal(msg['msg_type'], msg_type) + nt.assert_equal(msg['msg_type'], msg_type) if parent: - yield nt.assert_equal(msg['parent_header']['msg_id'], parent) + nt.assert_equal(msg['parent_header']['msg_id'], parent) content = msg['content'] ref = references[msg['msg_type']] - for tst in ref.check(content): - yield tst + ref.check(content) #----------------------------------------------------------------------------- @@ -297,54 +237,47 @@ def validate_message(msg, msg_type=None, parent=None): # Shell channel -@dec.parametric def test_execute(): flush_channels() msg_id = KC.execute(code='x=1') reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'execute_reply', msg_id): - yield tst + validate_message(reply, 'execute_reply', msg_id) -@dec.parametric def test_execute_silent(): flush_channels() msg_id, reply = execute(code='x=1', silent=True) # flush status=idle status = KC.iopub_channel.get_msg(timeout=TIMEOUT) - for tst in validate_message(status, 'status', msg_id): - yield tst + validate_message(status, 'status', msg_id) nt.assert_equal(status['content']['execution_state'], 'idle') - yield nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1) + nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1) count = reply['execution_count'] msg_id, reply = execute(code='x=2', silent=True) # flush status=idle status = KC.iopub_channel.get_msg(timeout=TIMEOUT) - for tst in validate_message(status, 'status', msg_id): - yield tst - yield nt.assert_equal(status['content']['execution_state'], 'idle') + validate_message(status, 'status', msg_id) + nt.assert_equal(status['content']['execution_state'], 'idle') - yield nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1) + nt.assert_raises(Empty, KC.iopub_channel.get_msg, timeout=0.1) count_2 = reply['execution_count'] - yield nt.assert_equal(count_2, count) + nt.assert_equal(count_2, count) -@dec.parametric def test_execute_error(): flush_channels() msg_id, reply = execute(code='1/0') - yield nt.assert_equal(reply['status'], 'error') - yield nt.assert_equal(reply['ename'], 'ZeroDivisionError') + nt.assert_equal(reply['status'], 'error') + nt.assert_equal(reply['ename'], 'ZeroDivisionError') pyerr = KC.iopub_channel.get_msg(timeout=TIMEOUT) - for tst in validate_message(pyerr, 'pyerr', msg_id): - yield tst + validate_message(pyerr, 'pyerr', msg_id) def test_execute_inc(): @@ -405,17 +338,14 @@ def test_user_expressions_fail(): nt.assert_equal(foo['ename'], 'NameError') -@dec.parametric def test_oinfo(): flush_channels() msg_id = KC.object_info('a') reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'object_info_reply', msg_id): - yield tst + validate_message(reply, 'object_info_reply', msg_id) -@dec.parametric def test_oinfo_found(): flush_channels() @@ -423,15 +353,13 @@ def test_oinfo_found(): msg_id = KC.object_info('a') reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'object_info_reply', msg_id): - yield tst + validate_message(reply, 'object_info_reply', msg_id) content = reply['content'] - yield nt.assert_true(content['found']) + assert content['found'] argspec = content['argspec'] - yield nt.assert_true(argspec is None, "didn't expect argspec dict, got %r" % argspec) + nt.assert_is(argspec, None) -@dec.parametric def test_oinfo_detail(): flush_channels() @@ -439,28 +367,24 @@ def test_oinfo_detail(): msg_id = KC.object_info('ip.object_inspect', detail_level=2) reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'object_info_reply', msg_id): - yield tst + validate_message(reply, 'object_info_reply', msg_id) content = reply['content'] - yield nt.assert_true(content['found']) + assert content['found'] argspec = content['argspec'] - yield nt.assert_true(isinstance(argspec, dict), "expected non-empty argspec dict, got %r" % argspec) - yield nt.assert_equal(argspec['defaults'], [0]) + nt.assert_is_instance(argspec, dict, "expected non-empty argspec dict, got %r" % argspec) + nt.assert_equal(argspec['defaults'], [0]) -@dec.parametric def test_oinfo_not_found(): flush_channels() msg_id = KC.object_info('dne') reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'object_info_reply', msg_id): - yield tst + validate_message(reply, 'object_info_reply', msg_id) content = reply['content'] - yield nt.assert_false(content['found']) + nt.assert_false(content['found']) -@dec.parametric def test_complete(): flush_channels() @@ -468,49 +392,42 @@ def test_complete(): msg_id = KC.complete('al', 'al', 2) reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'complete_reply', msg_id): - yield tst + validate_message(reply, 'complete_reply', msg_id) matches = reply['content']['matches'] for name in ('alpha', 'albert'): - yield nt.assert_true(name in matches, "Missing match: %r" % name) + nt.assert_in(name, matches) -@dec.parametric def test_kernel_info_request(): flush_channels() msg_id = KC.kernel_info() reply = KC.get_shell_msg(timeout=TIMEOUT) - for tst in validate_message(reply, 'kernel_info_reply', msg_id): - yield tst + validate_message(reply, 'kernel_info_reply', msg_id) # IOPub channel -@dec.parametric def test_stream(): flush_channels() msg_id, reply = execute("print('hi')") stdout = KC.iopub_channel.get_msg(timeout=TIMEOUT) - for tst in validate_message(stdout, 'stream', msg_id): - yield tst + validate_message(stdout, 'stream', msg_id) content = stdout['content'] - yield nt.assert_equal(content['name'], u'stdout') - yield nt.assert_equal(content['data'], u'hi\n') + nt.assert_equal(content['name'], u'stdout') + nt.assert_equal(content['data'], u'hi\n') -@dec.parametric def test_display_data(): flush_channels() msg_id, reply = execute("from IPython.core.display import display; display(1)") display = KC.iopub_channel.get_msg(timeout=TIMEOUT) - for tst in validate_message(display, 'display_data', parent=msg_id): - yield tst + validate_message(display, 'display_data', parent=msg_id) data = display['content']['data'] - yield nt.assert_equal(data['text/plain'], u'1') + nt.assert_equal(data['text/plain'], u'1') diff --git a/IPython/kernel/tests/test_multikernelmanager.py b/IPython/kernel/tests/test_multikernelmanager.py index 0c6ffe5..3838b72 100644 --- a/IPython/kernel/tests/test_multikernelmanager.py +++ b/IPython/kernel/tests/test_multikernelmanager.py @@ -7,7 +7,7 @@ from unittest import TestCase from IPython.testing import decorators as dec from IPython.config.loader import Config -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.kernel import KernelManager from IPython.kernel.multikernelmanager import MultiKernelManager @@ -31,20 +31,13 @@ class TestKernelManager(TestCase): self.assertTrue(kid in km) self.assertTrue(kid in km.list_kernel_ids()) self.assertEqual(len(km),1) - km.restart_kernel(kid) + km.restart_kernel(kid, now=True) self.assertTrue(km.is_alive(kid)) self.assertTrue(kid in km.list_kernel_ids()) - # We need a delay here to give the restarting kernel a chance to - # restart. Otherwise, the interrupt will kill it, causing the test - # suite to hang. The reason it *hangs* is that the shutdown - # message for the restart sometimes hasn't been sent to the kernel. - # Because linger is oo on the shell channel, the context can't - # close until the message is sent to the kernel, which is not dead. - time.sleep(1.0) km.interrupt_kernel(kid) k = km.get_kernel(kid) self.assertTrue(isinstance(k, KernelManager)) - km.shutdown_kernel(kid) + km.shutdown_kernel(kid, now=True) self.assertTrue(not kid in km) def _run_cinfo(self, km, transport, ip): @@ -63,7 +56,7 @@ class TestKernelManager(TestCase): self.assertTrue('hb_port' in cinfo) stream = km.connect_hb(kid) stream.close() - km.shutdown_kernel(kid) + km.shutdown_kernel(kid, now=True) def test_tcp_lifecycle(self): km = self._get_tcp_km() @@ -71,7 +64,7 @@ class TestKernelManager(TestCase): def test_tcp_cinfo(self): km = self._get_tcp_km() - self._run_cinfo(km, 'tcp', LOCALHOST) + self._run_cinfo(km, 'tcp', localhost()) @dec.skip_win32 def test_ipc_lifecycle(self): diff --git a/IPython/kernel/tests/test_public_api.py b/IPython/kernel/tests/test_public_api.py index 39cd7e7..eee44c0 100644 --- a/IPython/kernel/tests/test_public_api.py +++ b/IPython/kernel/tests/test_public_api.py @@ -14,8 +14,6 @@ Authors import nose.tools as nt -from IPython.testing import decorators as dec - from IPython.kernel import launcher, connect from IPython import kernel @@ -23,25 +21,21 @@ from IPython import kernel # Classes and functions #----------------------------------------------------------------------------- -@dec.parametric def test_kms(): for base in ("", "Multi"): KM = base + "KernelManager" - yield nt.assert_true(KM in dir(kernel), KM) + nt.assert_in(KM, dir(kernel)) -@dec.parametric def test_kcs(): for base in ("", "Blocking"): KM = base + "KernelClient" - yield nt.assert_true(KM in dir(kernel), KM) + nt.assert_in(KM, dir(kernel)) -@dec.parametric def test_launcher(): for name in launcher.__all__: - yield nt.assert_true(name in dir(kernel), name) + nt.assert_in(name, dir(kernel)) -@dec.parametric def test_connect(): for name in connect.__all__: - yield nt.assert_true(name in dir(kernel), name) + nt.assert_in(name, dir(kernel)) diff --git a/IPython/kernel/tests/utils.py b/IPython/kernel/tests/utils.py new file mode 100644 index 0000000..a3cd92f --- /dev/null +++ b/IPython/kernel/tests/utils.py @@ -0,0 +1,167 @@ +"""utilities for testing IPython kernels""" + +#------------------------------------------------------------------------------- +# Copyright (C) 2013 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#------------------------------------------------------------------------------- + +#------------------------------------------------------------------------------- +# Imports +#------------------------------------------------------------------------------- + +import atexit + +from contextlib import contextmanager +from subprocess import PIPE +from Queue import Empty + +import nose.tools as nt + +from IPython.kernel import KernelManager + +#------------------------------------------------------------------------------- +# Globals +#------------------------------------------------------------------------------- + +STARTUP_TIMEOUT = 60 +TIMEOUT = 15 + +KM = None +KC = None + +#------------------------------------------------------------------------------- +# code +#------------------------------------------------------------------------------- + + +def start_new_kernel(): + """start a new kernel, and return its Manager and Client""" + km = KernelManager() + km.start_kernel(stdout=PIPE, stderr=PIPE) + kc = km.client() + kc.start_channels() + + msg_id = kc.kernel_info() + kc.get_shell_msg(block=True, timeout=STARTUP_TIMEOUT) + flush_channels(kc) + return km, kc + +def flush_channels(kc=None): + """flush any messages waiting on the queue""" + from .test_message_spec import validate_message + + if kc is None: + kc = KC + for channel in (kc.shell_channel, kc.iopub_channel): + while True: + try: + msg = channel.get_msg(block=True, timeout=0.1) + except Empty: + break + else: + validate_message(msg) + + +def execute(code='', kc=None, **kwargs): + """wrapper for doing common steps for validating an execution request""" + from .test_message_spec import validate_message + if kc is None: + kc = KC + msg_id = kc.execute(code=code, **kwargs) + reply = kc.get_shell_msg(timeout=TIMEOUT) + validate_message(reply, 'execute_reply', msg_id) + busy = kc.get_iopub_msg(timeout=TIMEOUT) + validate_message(busy, 'status', msg_id) + nt.assert_equal(busy['content']['execution_state'], 'busy') + + if not kwargs.get('silent'): + pyin = kc.get_iopub_msg(timeout=TIMEOUT) + validate_message(pyin, 'pyin', msg_id) + nt.assert_equal(pyin['content']['code'], code) + + return msg_id, reply['content'] + +def start_global_kernel(): + """start the global kernel (if it isn't running) and return its client""" + global KM, KC + if KM is None: + KM, KC = start_new_kernel() + atexit.register(stop_global_kernel) + return KC + +@contextmanager +def kernel(): + """Context manager for the global kernel instance + + Should be used for most kernel tests + + Returns + ------- + kernel_client: connected KernelClient instance + """ + yield start_global_kernel() + +def uses_kernel(test_f): + """Decorator for tests that use the global kernel""" + def wrapped_test(): + with kernel() as kc: + test_f(kc) + wrapped_test.__doc__ = test_f.__doc__ + wrapped_test.__name__ = test_f.__name__ + return wrapped_test + +def stop_global_kernel(): + """Stop the global shared kernel instance, if it exists""" + global KM, KC + KC.stop_channels() + KC = None + if KM is None: + return + KM.shutdown_kernel(now=True) + KM = None + +@contextmanager +def new_kernel(): + """Context manager for a new kernel in a subprocess + + Should only be used for tests where the kernel must not be re-used. + + Returns + ------- + kernel_client: connected KernelClient instance + """ + km, kc = start_new_kernel() + try: + yield kc + finally: + kc.stop_channels() + km.shutdown_kernel(now=True) + + +def assemble_output(iopub): + """assemble stdout/err from an execution""" + stdout = '' + stderr = '' + while True: + msg = iopub.get_msg(block=True, timeout=1) + msg_type = msg['msg_type'] + content = msg['content'] + if msg_type == 'status' and content['execution_state'] == 'idle': + # idle message signals end of output + break + elif msg['msg_type'] == 'stream': + if content['name'] == 'stdout': + stdout += content['data'] + elif content['name'] == 'stderr': + stderr += content['data'] + else: + raise KeyError("bad stream: %r" % content['name']) + else: + # other output, ignored + pass + return stdout, stderr + + + diff --git a/IPython/kernel/zmq/heartbeat.py b/IPython/kernel/zmq/heartbeat.py index a018abb..ccfd5dc 100644 --- a/IPython/kernel/zmq/heartbeat.py +++ b/IPython/kernel/zmq/heartbeat.py @@ -19,7 +19,7 @@ from threading import Thread import zmq -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost #----------------------------------------------------------------------------- # Code @@ -29,7 +29,9 @@ from IPython.utils.localinterfaces import LOCALHOST class Heartbeat(Thread): "A simple ping-pong style heartbeat that runs in a thread." - def __init__(self, context, addr=('tcp', LOCALHOST, 0)): + def __init__(self, context, addr=None): + if addr is None: + addr = ('tcp', localhost(), 0) Thread.__init__(self) self.context = context self.transport, self.ip, self.port = addr diff --git a/IPython/kernel/zmq/ipkernel.py b/IPython/kernel/zmq/ipkernel.py index 8445ec4..ec0d8be 100755 --- a/IPython/kernel/zmq/ipkernel.py +++ b/IPython/kernel/zmq/ipkernel.py @@ -87,7 +87,7 @@ class Kernel(Configurable): if self.shell is not None: self.shell.user_module = new - user_ns = Dict(default_value=None) + user_ns = Instance(dict, args=None, allow_none=True) def _user_ns_changed(self, name, old, new): if self.shell is not None: self.shell.user_ns = new @@ -131,6 +131,7 @@ class Kernel(Configurable): # A reference to the Python builtin 'raw_input' function. # (i.e., __builtin__.raw_input for Python 2.7, builtins.input for Python 3) _sys_raw_input = Any() + _sys_eval_input = Any() # set of aborted msg_ids aborted = Set() @@ -352,15 +353,18 @@ class Kernel(Configurable): # raw_input in the user namespace. if content.get('allow_stdin', False): raw_input = lambda prompt='': self._raw_input(prompt, ident, parent) + input = lambda prompt='': eval(raw_input(prompt)) else: - raw_input = lambda prompt='' : self._no_raw_input() + raw_input = input = lambda prompt='' : self._no_raw_input() if py3compat.PY3: self._sys_raw_input = __builtin__.input __builtin__.input = raw_input else: self._sys_raw_input = __builtin__.raw_input + self._sys_eval_input = __builtin__.input __builtin__.raw_input = raw_input + __builtin__.input = input # Set the parent message of the display hook and out streams. shell.displayhook.set_parent(parent) @@ -403,6 +407,7 @@ class Kernel(Configurable): __builtin__.input = self._sys_raw_input else: __builtin__.raw_input = self._sys_raw_input + __builtin__.input = self._sys_eval_input reply_content[u'status'] = status diff --git a/IPython/kernel/zmq/kernelapp.py b/IPython/kernel/zmq/kernelapp.py index 3b100fd..7461bbf 100644 --- a/IPython/kernel/zmq/kernelapp.py +++ b/IPython/kernel/zmq/kernelapp.py @@ -39,7 +39,7 @@ from IPython.core.shellapp import ( InteractiveShellApp, shell_flags, shell_aliases ) from IPython.utils import io -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.path import filefind from IPython.utils.py3compat import str_to_bytes from IPython.utils.traitlets import ( @@ -156,7 +156,8 @@ class IPKernelApp(BaseIPythonApplication, InteractiveShellApp): else: return 'kernel-ipc' else: - return LOCALHOST + return localhost() + hb_port = Integer(0, config=True, help="set the heartbeat port [default: random]") shell_port = Integer(0, config=True, help="set the shell (ROUTER) port [default: random]") iopub_port = Integer(0, config=True, help="set the iopub (PUB) port [default: random]") diff --git a/IPython/kernel/zmq/session.py b/IPython/kernel/zmq/session.py index b71d043..2ba4738 100644 --- a/IPython/kernel/zmq/session.py +++ b/IPython/kernel/zmq/session.py @@ -436,8 +436,15 @@ class Session(Configurable): msg = dict(a=[1,'hi']) try: packed = pack(msg) - except Exception: - raise ValueError("packer could not serialize a simple message") + except Exception as e: + msg = "packer '{packer}' could not serialize a simple message: {e}{jsonmsg}" + if self.packer == 'json': + jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod + else: + jsonmsg = "" + raise ValueError( + msg.format(packer=self.packer, e=e, jsonmsg=jsonmsg) + ) # ensure packed message is bytes if not isinstance(packed, bytes): @@ -446,8 +453,16 @@ class Session(Configurable): # check that unpack is pack's inverse try: unpacked = unpack(packed) - except Exception: - raise ValueError("unpacker could not handle the packer's output") + assert unpacked == msg + except Exception as e: + msg = "unpacker '{unpacker}' could not handle output from packer '{packer}': {e}{jsonmsg}" + if self.packer == 'json': + jsonmsg = "\nzmq.utils.jsonapi.jsonmod = %s" % jsonapi.jsonmod + else: + jsonmsg = "" + raise ValueError( + msg.format(packer=self.packer, unpacker=self.unpacker, e=e, jsonmsg=jsonmsg) + ) # check datetime support msg = dict(t=datetime.now()) diff --git a/IPython/kernel/zmq/tests/test_serialize.py b/IPython/kernel/zmq/tests/test_serialize.py index 65d5b28..c5f0394 100644 --- a/IPython/kernel/zmq/tests/test_serialize.py +++ b/IPython/kernel/zmq/tests/test_serialize.py @@ -46,7 +46,6 @@ DTYPES = ('uint8', 'float64', 'int32', [('g', 'float32')], '|S10') # Tests #------------------------------------------------------------------------------- -@dec.parametric def test_roundtrip_simple(): for obj in [ 'hello', @@ -55,18 +54,16 @@ def test_roundtrip_simple(): (b'123', 'hello'), ]: obj2 = roundtrip(obj) - yield nt.assert_equals(obj, obj2) + nt.assert_equal(obj, obj2) -@dec.parametric def test_roundtrip_nested(): for obj in [ dict(a=range(5), b={1:b'hello'}), [range(5),[range(3),(1,[b'whoda'])]], ]: obj2 = roundtrip(obj) - yield nt.assert_equals(obj, obj2) + nt.assert_equal(obj, obj2) -@dec.parametric def test_roundtrip_buffered(): for obj in [ dict(a=b"x"*1025), @@ -74,10 +71,10 @@ def test_roundtrip_buffered(): [b"hello"*501, 1,2,3] ]: bufs = serialize_object(obj) - yield nt.assert_equals(len(bufs), 2) + nt.assert_equal(len(bufs), 2) obj2, remainder = unserialize_object(bufs) - yield nt.assert_equals(remainder, []) - yield nt.assert_equals(obj, obj2) + nt.assert_equal(remainder, []) + nt.assert_equal(obj, obj2) def _scrub_nan(A): """scrub nans out of empty arrays @@ -93,7 +90,6 @@ def _scrub_nan(A): # e.g. str dtype pass -@dec.parametric @dec.skip_without('numpy') def test_numpy(): import numpy @@ -104,12 +100,11 @@ def test_numpy(): _scrub_nan(A) bufs = serialize_object(A) B, r = unserialize_object(bufs) - yield nt.assert_equals(r, []) - yield nt.assert_equals(A.shape, B.shape) - yield nt.assert_equals(A.dtype, B.dtype) - yield assert_array_equal(A,B) + nt.assert_equal(r, []) + nt.assert_equal(A.shape, B.shape) + nt.assert_equal(A.dtype, B.dtype) + assert_array_equal(A,B) -@dec.parametric @dec.skip_without('numpy') def test_recarray(): import numpy @@ -124,12 +119,11 @@ def test_recarray(): bufs = serialize_object(A) B, r = unserialize_object(bufs) - yield nt.assert_equals(r, []) - yield nt.assert_equals(A.shape, B.shape) - yield nt.assert_equals(A.dtype, B.dtype) - yield assert_array_equal(A,B) + nt.assert_equal(r, []) + nt.assert_equal(A.shape, B.shape) + nt.assert_equal(A.dtype, B.dtype) + assert_array_equal(A,B) -@dec.parametric @dec.skip_without('numpy') def test_numpy_in_seq(): import numpy @@ -140,15 +134,14 @@ def test_numpy_in_seq(): _scrub_nan(A) bufs = serialize_object((A,1,2,b'hello')) canned = pickle.loads(bufs[0]) - yield nt.assert_true(canned[0], CannedArray) + nt.assert_is_instance(canned[0], CannedArray) tup, r = unserialize_object(bufs) B = tup[0] - yield nt.assert_equals(r, []) - yield nt.assert_equals(A.shape, B.shape) - yield nt.assert_equals(A.dtype, B.dtype) - yield assert_array_equal(A,B) + nt.assert_equal(r, []) + nt.assert_equal(A.shape, B.shape) + nt.assert_equal(A.dtype, B.dtype) + assert_array_equal(A,B) -@dec.parametric @dec.skip_without('numpy') def test_numpy_in_dict(): import numpy @@ -159,27 +152,25 @@ def test_numpy_in_dict(): _scrub_nan(A) bufs = serialize_object(dict(a=A,b=1,c=range(20))) canned = pickle.loads(bufs[0]) - yield nt.assert_true(canned['a'], CannedArray) + nt.assert_is_instance(canned['a'], CannedArray) d, r = unserialize_object(bufs) B = d['a'] - yield nt.assert_equals(r, []) - yield nt.assert_equals(A.shape, B.shape) - yield nt.assert_equals(A.dtype, B.dtype) - yield assert_array_equal(A,B) + nt.assert_equal(r, []) + nt.assert_equal(A.shape, B.shape) + nt.assert_equal(A.dtype, B.dtype) + assert_array_equal(A,B) -@dec.parametric def test_class(): @interactive class C(object): a=5 bufs = serialize_object(dict(C=C)) canned = pickle.loads(bufs[0]) - yield nt.assert_true(canned['C'], CannedClass) + nt.assert_is_instance(canned['C'], CannedClass) d, r = unserialize_object(bufs) C2 = d['C'] - yield nt.assert_equal(C2.a, C.a) + nt.assert_equal(C2.a, C.a) -@dec.parametric def test_class_oldstyle(): @interactive class C: @@ -187,42 +178,38 @@ def test_class_oldstyle(): bufs = serialize_object(dict(C=C)) canned = pickle.loads(bufs[0]) - yield nt.assert_true(isinstance(canned['C'], CannedClass)) + nt.assert_is_instance(canned['C'], CannedClass) d, r = unserialize_object(bufs) C2 = d['C'] - yield nt.assert_equal(C2.a, C.a) + nt.assert_equal(C2.a, C.a) -@dec.parametric def test_tuple(): tup = (lambda x:x, 1) bufs = serialize_object(tup) canned = pickle.loads(bufs[0]) - yield nt.assert_true(isinstance(canned, tuple)) + nt.assert_is_instance(canned, tuple) t2, r = unserialize_object(bufs) - yield nt.assert_equal(t2[0](t2[1]), tup[0](tup[1])) + nt.assert_equal(t2[0](t2[1]), tup[0](tup[1])) point = namedtuple('point', 'x y') -@dec.parametric def test_namedtuple(): p = point(1,2) bufs = serialize_object(p) canned = pickle.loads(bufs[0]) - yield nt.assert_true(isinstance(canned, point)) + nt.assert_is_instance(canned, point) p2, r = unserialize_object(bufs, globals()) - yield nt.assert_equal(p2.x, p.x) - yield nt.assert_equal(p2.y, p.y) + nt.assert_equal(p2.x, p.x) + nt.assert_equal(p2.y, p.y) -@dec.parametric def test_list(): lis = [lambda x:x, 1] bufs = serialize_object(lis) canned = pickle.loads(bufs[0]) - yield nt.assert_true(isinstance(canned, list)) + nt.assert_is_instance(canned, list) l2, r = unserialize_object(bufs) - yield nt.assert_equal(l2[0](l2[1]), lis[0](lis[1])) + nt.assert_equal(l2[0](l2[1]), lis[0](lis[1])) -@dec.parametric def test_class_inheritance(): @interactive class C(object): @@ -234,8 +221,8 @@ def test_class_inheritance(): bufs = serialize_object(dict(D=D)) canned = pickle.loads(bufs[0]) - yield nt.assert_true(canned['D'], CannedClass) + nt.assert_is_instance(canned['D'], CannedClass) d, r = unserialize_object(bufs) D2 = d['D'] - yield nt.assert_equal(D2.a, D.a) - yield nt.assert_equal(D2.b, D.b) + nt.assert_equal(D2.a, D.a) + nt.assert_equal(D2.b, D.b) diff --git a/IPython/kernel/zmq/tests/test_session.py b/IPython/kernel/zmq/tests/test_session.py index df7b77b..06b6d6f 100644 --- a/IPython/kernel/zmq/tests/test_session.py +++ b/IPython/kernel/zmq/tests/test_session.py @@ -20,6 +20,12 @@ from zmq.eventloop.zmqstream import ZMQStream from IPython.kernel.zmq import session as ss +def _bad_packer(obj): + raise TypeError("I don't work") + +def _bad_unpacker(bytes): + raise TypeError("I don't work either") + class SessionTestCase(BaseZMQTestCase): def setUp(self): @@ -222,4 +228,44 @@ class TestSession(SessionTestCase): self.assertTrue(len(session.digest_history) == 100) session._add_digest(uuid.uuid4().bytes) self.assertTrue(len(session.digest_history) == 91) - + + def test_bad_pack(self): + try: + session = ss.Session(pack=_bad_packer) + except ValueError as e: + self.assertIn("could not serialize", str(e)) + self.assertIn("don't work", str(e)) + else: + self.fail("Should have raised ValueError") + + def test_bad_unpack(self): + try: + session = ss.Session(unpack=_bad_unpacker) + except ValueError as e: + self.assertIn("could not handle output", str(e)) + self.assertIn("don't work either", str(e)) + else: + self.fail("Should have raised ValueError") + + def test_bad_packer(self): + try: + session = ss.Session(packer=__name__ + '._bad_packer') + except ValueError as e: + self.assertIn("could not serialize", str(e)) + self.assertIn("don't work", str(e)) + else: + self.fail("Should have raised ValueError") + + def test_bad_unpacker(self): + try: + session = ss.Session(unpacker=__name__ + '._bad_unpacker') + except ValueError as e: + self.assertIn("could not handle output", str(e)) + self.assertIn("don't work either", str(e)) + else: + self.fail("Should have raised ValueError") + + def test_bad_roundtrip(self): + with self.assertRaises(ValueError): + session= ss.Session(unpack=lambda b: 5) + diff --git a/IPython/kernel/zmq/tests/test_start_kernel.py b/IPython/kernel/zmq/tests/test_start_kernel.py index 0ba37ce..a31c1f3 100644 --- a/IPython/kernel/zmq/tests/test_start_kernel.py +++ b/IPython/kernel/zmq/tests/test_start_kernel.py @@ -8,10 +8,38 @@ def test_ipython_start_kernel_userns(): cmd = ('from IPython import start_kernel\n' 'ns = {"tre": 123}\n' 'start_kernel(user_ns=ns)') - + with setup_kernel(cmd) as client: msg_id = client.object_info('tre') msg = client.get_shell_msg(block=True, timeout=TIMEOUT) content = msg['content'] assert content['found'] - nt.assert_equal(content['string_form'], u'123') \ No newline at end of file + nt.assert_equal(content['string_form'], u'123') + + # user_module should be an instance of DummyMod + msg_id = client.execute("usermod = get_ipython().user_module") + msg = client.get_shell_msg(block=True, timeout=TIMEOUT) + content = msg['content'] + nt.assert_equal(content['status'], u'ok') + msg_id = client.object_info('usermod') + msg = client.get_shell_msg(block=True, timeout=TIMEOUT) + content = msg['content'] + assert content['found'] + nt.assert_in('DummyMod', content['string_form']) + +def test_ipython_start_kernel_no_userns(): + # Issue #4188 - user_ns should be passed to shell as None, not {} + cmd = ('from IPython import start_kernel\n' + 'start_kernel()') + + with setup_kernel(cmd) as client: + # user_module should not be an instance of DummyMod + msg_id = client.execute("usermod = get_ipython().user_module") + msg = client.get_shell_msg(block=True, timeout=TIMEOUT) + content = msg['content'] + nt.assert_equal(content['status'], u'ok') + msg_id = client.object_info('usermod') + msg = client.get_shell_msg(block=True, timeout=TIMEOUT) + content = msg['content'] + assert content['found'] + nt.assert_not_in('DummyMod', content['string_form']) diff --git a/IPython/kernel/zmq/zmqshell.py b/IPython/kernel/zmq/zmqshell.py index 27d8113..df7f5a5 100644 --- a/IPython/kernel/zmq/zmqshell.py +++ b/IPython/kernel/zmq/zmqshell.py @@ -86,14 +86,11 @@ class ZMQDisplayPublisher(DisplayPublisher): parent=self.parent_header, ident=self.topic, ) - def clear_output(self, stdout=True, stderr=True, other=True): - content = dict(stdout=stdout, stderr=stderr, other=other) - - if stdout: - print('\r', file=sys.stdout, end='') - if stderr: - print('\r', file=sys.stderr, end='') - + def clear_output(self, wait=False): + content = dict(wait=wait) + + print('\r', file=sys.stdout, end='') + print('\r', file=sys.stderr, end='') self._flush_streams() self.session.send( diff --git a/IPython/lib/demo.py b/IPython/lib/demo.py index 621e44a..0c1fa8e 100644 --- a/IPython/lib/demo.py +++ b/IPython/lib/demo.py @@ -14,14 +14,14 @@ The classes are (see their docstrings for further details): - Demo: pure python demos - IPythonDemo: demos with input to be processed by IPython as if it had been - typed interactively (so magics work, as well as any other special syntax you - may have added via input prefilters). + typed interactively (so magics work, as well as any other special syntax you + may have added via input prefilters). - LineDemo: single-line version of the Demo class. These demos are executed - one line at a time, and require no markup. + one line at a time, and require no markup. - IPythonLineDemo: IPython version of the LineDemo class (the demo is - executed a line at a time, but processed via IPython). + executed a line at a time, but processed via IPython). - ClearMixin: mixin to make Demo classes with less visual clutter. It declares an empty marquee and a pre_cmd that clears the screen before each @@ -214,18 +214,18 @@ class Demo(object): Optional inputs: - title: a string to use as the demo name. Of most use when the demo - you are making comes from an object that has no filename, or if you - want an alternate denotation distinct from the filename. + you are making comes from an object that has no filename, or if you + want an alternate denotation distinct from the filename. - arg_str(''): a string of arguments, internally converted to a list - just like sys.argv, so the demo script can see a similar - environment. + just like sys.argv, so the demo script can see a similar + environment. - auto_all(None): global flag to run all blocks automatically without - confirmation. This attribute overrides the block-level tags and - applies to the whole demo. It is an attribute of the object, and - can be changed at runtime simply by reassigning it to a boolean - value. + confirmation. This attribute overrides the block-level tags and + applies to the whole demo. It is an attribute of the object, and + can be changed at runtime simply by reassigning it to a boolean + value. """ if hasattr(src, "read"): # It seems to be a file or a file-like object diff --git a/IPython/lib/inputhook.py b/IPython/lib/inputhook.py index 93d090b..5ae158c 100644 --- a/IPython/lib/inputhook.py +++ b/IPython/lib/inputhook.py @@ -481,6 +481,19 @@ set_inputhook = inputhook_manager.set_inputhook current_gui = inputhook_manager.current_gui clear_app_refs = inputhook_manager.clear_app_refs +guis = {None: clear_inputhook, + GUI_NONE: clear_inputhook, + GUI_OSX: lambda app=False: None, + GUI_TK: enable_tk, + GUI_GTK: enable_gtk, + GUI_WX: enable_wx, + GUI_QT: enable_qt4, # qt3 not supported + GUI_QT4: enable_qt4, + GUI_GLUT: enable_glut, + GUI_PYGLET: enable_pyglet, + GUI_GTK3: enable_gtk3, +} + # Convenience function to switch amongst them def enable_gui(gui=None, app=None): @@ -507,18 +520,6 @@ def enable_gui(gui=None, app=None): PyOS_InputHook wrapper object or the GUI toolkit app created, if there was one. """ - guis = {None: clear_inputhook, - GUI_NONE: clear_inputhook, - GUI_OSX: lambda app=False: None, - GUI_TK: enable_tk, - GUI_GTK: enable_gtk, - GUI_WX: enable_wx, - GUI_QT: enable_qt4, # qt3 not supported - GUI_QT4: enable_qt4, - GUI_GLUT: enable_glut, - GUI_PYGLET: enable_pyglet, - GUI_GTK3: enable_gtk3, - } try: gui_hook = guis[gui] except KeyError: diff --git a/IPython/lib/irunner.py b/IPython/lib/irunner.py index 623cdb4..e8c65f2 100755 --- a/IPython/lib/irunner.py +++ b/IPython/lib/irunner.py @@ -93,9 +93,9 @@ class InteractiveRunner(object): - program: command to execute the given program. - prompts: a list of patterns to match as valid prompts, in the - format used by pexpect. This basically means that it can be either - a string (to be compiled as a regular expression) or a list of such - (it must be a true list, as pexpect does type checks). + format used by pexpect. This basically means that it can be either + a string (to be compiled as a regular expression) or a list of such + (it must be a true list, as pexpect does type checks). If more than one prompt is given, the first is treated as the main program prompt and the others as 'continuation' prompts, like @@ -107,19 +107,19 @@ class InteractiveRunner(object): Optional inputs: - args(None): optional list of strings to pass as arguments to the - child program. + child program. - out(sys.stdout): if given, an output stream to be used when writing - output. The only requirement is that it must have a .write() method. + output. The only requirement is that it must have a .write() method. Public members not parameterized in the constructor: - delaybeforesend(0): Newer versions of pexpect have a delay before - sending each new input. For our purposes here, it's typically best - to just set this to zero, but if you encounter reliability problems - or want an interactive run to pause briefly at each prompt, just - increase this value (it is measured in seconds). Note that this - variable is not honored at all by older versions of pexpect. + sending each new input. For our purposes here, it's typically best + to just set this to zero, but if you encounter reliability problems + or want an interactive run to pause briefly at each prompt, just + increase this value (it is measured in seconds). Note that this + variable is not honored at all by older versions of pexpect. """ self.program = program @@ -154,7 +154,7 @@ class InteractiveRunner(object): Inputs: - -fname: name of the file to execute. + - fname: name of the file to execute. See the run_source docstring for the meaning of the optional arguments.""" @@ -173,15 +173,15 @@ class InteractiveRunner(object): Inputs: - source: a string of code to be executed, or an open file object we - can iterate over. + can iterate over. Optional inputs: - interact(False): if true, start to interact with the running - program at the end of the script. Otherwise, just exit. + program at the end of the script. Otherwise, just exit. - get_output(False): if true, capture the output of the child process - (filtering the input commands out) and return it as a string. + (filtering the input commands out) and return it as a string. Returns: A string containing the process output, but only if requested. diff --git a/IPython/lib/pretty.py b/IPython/lib/pretty.py index 36d7824..2fee84c 100644 --- a/IPython/lib/pretty.py +++ b/IPython/lib/pretty.py @@ -229,7 +229,17 @@ class PrettyPrinter(_PrettyPrinterBase): self.buffer.append(Breakable(sep, width, self)) self.buffer_width += width self._break_outer_groups() - + + def break_(self): + """ + Explicitly insert a newline into the output, maintaining correct indentation. + """ + self.flush() + self.output.write(self.newline) + self.output.write(' ' * self.indentation) + self.output_width = self.indentation + self.buffer_width = 0 + def begin_group(self, indent=0, open=''): """ @@ -478,8 +488,12 @@ def _default_pprint(obj, p, cycle): """ klass = getattr(obj, '__class__', None) or type(obj) if getattr(klass, '__repr__', None) not in _baseclass_reprs: - # A user-provided repr. - p.text(repr(obj)) + # A user-provided repr. Find newlines and replace them with p.break_() + output = repr(obj) + for idx,output_line in enumerate(output.splitlines()): + if idx: + p.break_() + p.text(output_line) return p.begin_group(1, '<') p.pretty(klass) diff --git a/IPython/lib/tests/test_pretty.py b/IPython/lib/tests/test_pretty.py index 689ae48..2e0aaab 100644 --- a/IPython/lib/tests/test_pretty.py +++ b/IPython/lib/tests/test_pretty.py @@ -58,6 +58,23 @@ class NoModule(object): NoModule.__module__ = None +class Breaking(object): + def _repr_pretty_(self, p, cycle): + with p.group(4,"TG: ",":"): + p.text("Breaking(") + p.break_() + p.text(")") + +class BreakingRepr(object): + def __repr__(self): + return "Breaking(\n)" + +class BreakingReprParent(object): + def _repr_pretty_(self, p, cycle): + with p.group(4,"TG: ",":"): + p.pretty(BreakingRepr()) + + def test_indentation(): """Test correct indentation in groups""" @@ -118,3 +135,19 @@ def test_pprint_nomod(): """ output = pretty.pretty(NoModule) nt.assert_equal(output, 'NoModule') + +def test_pprint_break(): + """ + Test that p.break_ produces expected output + """ + output = pretty.pretty(Breaking()) + expected = "TG: Breaking(\n ):" + nt.assert_equal(output, expected) + +def test_pprint_break_repr(): + """ + Test that p.break_ is used in repr + """ + output = pretty.pretty(BreakingReprParent()) + expected = "TG: Breaking(\n ):" + nt.assert_equal(output, expected) \ No newline at end of file diff --git a/IPython/nbconvert/exporters/__init__.py b/IPython/nbconvert/exporters/__init__.py old mode 100755 new mode 100644 index c4eb2cb..23f6195 --- a/IPython/nbconvert/exporters/__init__.py +++ b/IPython/nbconvert/exporters/__init__.py @@ -1,8 +1,9 @@ from .export import * from .html import HTMLExporter from .slides import SlidesExporter -from .exporter import Exporter +from .templateexporter import TemplateExporter from .latex import LatexExporter from .markdown import MarkdownExporter from .python import PythonExporter from .rst import RSTExporter +from .exporter import Exporter diff --git a/IPython/nbconvert/exporters/export.py b/IPython/nbconvert/exporters/export.py old mode 100755 new mode 100644 index 31820e4..895579b --- a/IPython/nbconvert/exporters/export.py +++ b/IPython/nbconvert/exporters/export.py @@ -19,6 +19,7 @@ from IPython.nbformat.v3.nbbase import NotebookNode from IPython.config import Config from .exporter import Exporter +from .templateexporter import TemplateExporter from .html import HTMLExporter from .slides import SlidesExporter from .latex import LatexExporter @@ -122,7 +123,7 @@ def export(exporter, nb, **kw): return output, resources exporter_map = dict( - custom=Exporter, + custom=TemplateExporter, html=HTMLExporter, slides=SlidesExporter, latex=LatexExporter, diff --git a/IPython/nbconvert/exporters/exporter.py b/IPython/nbconvert/exporters/exporter.py old mode 100755 new mode 100644 index af9da29..b1126b4 --- a/IPython/nbconvert/exporters/exporter.py +++ b/IPython/nbconvert/exporters/exporter.py @@ -19,56 +19,21 @@ from __future__ import print_function, absolute_import # Stdlib imports import io import os -import inspect import copy import collections import datetime -# other libs/dependencies -from jinja2 import Environment, FileSystemLoader, ChoiceLoader, TemplateNotFound # IPython imports from IPython.config.configurable import LoggingConfigurable from IPython.config import Config from IPython.nbformat import current as nbformat -from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict, Any +from IPython.utils.traitlets import MetaHasTraits, Unicode, List from IPython.utils.importstring import import_item -from IPython.utils.text import indent -from IPython.utils import py3compat +from IPython.utils import text, py3compat from IPython.nbconvert import preprocessors as nbpreprocessors -from IPython.nbconvert import filters -#----------------------------------------------------------------------------- -# Globals and constants -#----------------------------------------------------------------------------- - -#Jinja2 extensions to load. -JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] - -default_filters = { - 'indent': indent, - 'markdown2html': filters.markdown2html, - 'ansi2html': filters.ansi2html, - 'filter_data_type': filters.DataTypeFilter, - 'get_lines': filters.get_lines, - 'highlight2html': filters.highlight2html, - 'highlight2latex': filters.highlight2latex, - 'ipython2python': filters.ipython2python, - 'posix_path': filters.posix_path, - 'markdown2latex': filters.markdown2latex, - 'markdown2rst': filters.markdown2rst, - 'comment_lines': filters.comment_lines, - 'strip_ansi': filters.strip_ansi, - 'strip_dollars': filters.strip_dollars, - 'strip_files_prefix': filters.strip_files_prefix, - 'html2text' : filters.html2text, - 'add_anchor': filters.add_anchor, - 'ansi2latex': filters.ansi2latex, - 'strip_math_space': filters.strip_math_space, - 'wrap_text': filters.wrap_text, - 'escape_latex': filters.escape_latex, -} #----------------------------------------------------------------------------- # Class @@ -81,70 +46,21 @@ class ResourcesDict(collections.defaultdict): class Exporter(LoggingConfigurable): """ - Exports notebooks into other file formats. Uses Jinja 2 templating engine - to output new formats. Inherit from this class if you are creating a new - template type along with new filters/preprocessors. If the filters/ - preprocessors provided by default suffice, there is no need to inherit from - this class. Instead, override the template_file and file_extension - traits via a config file. - - {filters} + Class containing methods that sequentially run a list of preprocessors on a + NotebookNode object and then return the modified NotebookNode object and + accompanying resources dict. """ - - # finish the docstring - __doc__ = __doc__.format(filters = '- '+'\n - '.join(default_filters.keys())) - - - template_file = Unicode(u'default', - config=True, - help="Name of the template file to use") - def _template_file_changed(self, name, old, new): - if new=='default': - self.template_file = self.default_template - else: - self.template_file = new - self.template = None - self._load_template() - - default_template = Unicode(u'') - template = Any() - environment = Any() file_extension = Unicode( - 'txt', config=True, + 'txt', config=True, help="Extension of the file that should be written to disk" ) - template_path = List(['.'], config=True) - def _template_path_changed(self, name, old, new): - self._load_template() - - default_template_path = Unicode( - os.path.join("..", "templates"), - help="Path where the template files are located.") - - template_skeleton_path = Unicode( - os.path.join("..", "templates", "skeleton"), - help="Path where the template skeleton files are located.") - - #Jinja block definitions - jinja_comment_block_start = Unicode("", config=True) - jinja_comment_block_end = Unicode("", config=True) - jinja_variable_block_start = Unicode("", config=True) - jinja_variable_block_end = Unicode("", config=True) - jinja_logic_block_start = Unicode("", config=True) - jinja_logic_block_end = Unicode("", config=True) - - #Extension that the template files use. - template_extension = Unicode(".tpl", config=True) - #Configurability, allows the user to easily add filters and preprocessors. preprocessors = List(config=True, help="""List of preprocessors, by name or namespace, to enable.""") - filters = Dict(config=True, - help="""Dictionary of filters, by name and namespace, to add to the Jinja - environment.""") + _preprocessors = None default_preprocessors = List([nbpreprocessors.coalesce_streams, nbpreprocessors.SVG2PDFPreprocessor, @@ -152,42 +68,34 @@ class Exporter(LoggingConfigurable): nbpreprocessors.CSSHTMLHeaderPreprocessor, nbpreprocessors.RevealHelpPreprocessor, nbpreprocessors.LatexPreprocessor, - nbpreprocessors.SphinxPreprocessor], + nbpreprocessors.HighlightMagicsPreprocessor], config=True, help="""List of preprocessors available by default, by name, namespace, instance, or type.""") - def __init__(self, config=None, extra_loaders=None, **kw): + def __init__(self, config=None, **kw): """ Public constructor - + Parameters ---------- config : config User configuration instance. - extra_loaders : list[of Jinja Loaders] - ordered list of Jinja loader to find templates. Will be tried in order - before the default FileSystem ones. - template : str (optional, kw arg) - Template to use when exporting. """ if not config: config = self.default_config - + super(Exporter, self).__init__(config=config, **kw) #Init - self._init_template() - self._init_environment(extra_loaders=extra_loaders) self._init_preprocessors() - self._init_filters() @property def default_config(self): return Config() - + def _config_changed(self, name, old, new): """When setting config, make sure to start with our default_config""" c = self.default_config @@ -196,56 +104,18 @@ class Exporter(LoggingConfigurable): if c != old: self.config = c super(Exporter, self)._config_changed(name, old, c) - - - def _load_template(self): - """Load the Jinja template object from the template file - - This is a no-op if the template attribute is already defined, - or the Jinja environment is not setup yet. - - This is triggered by various trait changes that would change the template. - """ - if self.template is not None: - return - # called too early, do nothing - if self.environment is None: - return - # Try different template names during conversion. First try to load the - # template by name with extension added, then try loading the template - # as if the name is explicitly specified, then try the name as a - # 'flavor', and lastly just try to load the template by module name. - module_name = self.__module__.rsplit('.', 1)[-1] - try_names = [] - if self.template_file: - try_names.extend([ - self.template_file + self.template_extension, - self.template_file, - module_name + '_' + self.template_file + self.template_extension, - ]) - try_names.append(module_name + self.template_extension) - for try_name in try_names: - self.log.debug("Attempting to load template %s", try_name) - try: - self.template = self.environment.get_template(try_name) - except (TemplateNotFound, IOError): - pass - except Exception as e: - self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True) - else: - self.log.info("Loaded template %s", try_name) - break - + + def from_notebook_node(self, nb, resources=None, **kw): """ Convert a notebook from a notebook node instance. - + Parameters ---------- nb : Notebook node - resources : dict (**kw) - of additional resources that can be accessed read/write by - preprocessors and filters. + resources : dict (**kw) + of additional resources that can be accessed read/write by + preprocessors. """ nb_copy = copy.deepcopy(nb) resources = self._init_resources(resources) @@ -253,26 +123,20 @@ class Exporter(LoggingConfigurable): # Preprocess nb_copy, resources = self._preprocess(nb_copy, resources) - self._load_template() - - if self.template is not None: - output = self.template.render(nb=nb_copy, resources=resources) - else: - raise IOError('template file "%s" could not be found' % self.template_file) - return output, resources + return nb_copy, resources def from_filename(self, filename, resources=None, **kw): """ Convert a notebook from a notebook file. - + Parameters ---------- filename : str Full filename of the notebook file to open and convert. """ - #Pull the metadata from the filesystem. + # Pull the metadata from the filesystem. if resources is None: resources = ResourcesDict() if not 'metadata' in resources or resources['metadata'] == '': @@ -282,16 +146,16 @@ class Exporter(LoggingConfigurable): resources['metadata']['name'] = notebook_name modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) - resources['metadata']['modified_date'] = modified_date.strftime("%B %d, %Y") - + resources['metadata']['modified_date'] = modified_date.strftime(text.date_format) + with io.open(filename) as f: - return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw) + return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw) def from_file(self, file_stream, resources=None, **kw): """ Convert a notebook from a notebook file. - + Parameters ---------- file_stream : file-like object @@ -304,10 +168,10 @@ class Exporter(LoggingConfigurable): """ Register a preprocessor. Preprocessors are classes that act upon the notebook before it is - passed into the Jinja templating engine. Preprocessors are also + passed into the Jinja templating engine. preprocessors are also capable of passing additional information to the Jinja templating engine. - + Parameters ---------- preprocessor : preprocessor @@ -317,130 +181,43 @@ class Exporter(LoggingConfigurable): isclass = isinstance(preprocessor, type) constructed = not isclass - #Handle preprocessor's registration based on it's type + # Handle preprocessor's registration based on it's type if constructed and isinstance(preprocessor, py3compat.string_types): - #Preprocessor is a string, import the namespace and recursively call - #this register_preprocessor method + # Preprocessor is a string, import the namespace and recursively call + # this register_preprocessor method preprocessor_cls = import_item(preprocessor) return self.register_preprocessor(preprocessor_cls, enabled) - + if constructed and hasattr(preprocessor, '__call__'): - #Preprocessor is a function, no need to construct it. - #Register and return the preprocessor. + # Preprocessor is a function, no need to construct it. + # Register and return the preprocessor. if enabled: preprocessor.enabled = True self._preprocessors.append(preprocessor) return preprocessor elif isclass and isinstance(preprocessor, MetaHasTraits): - #Preprocessor is configurable. Make sure to pass in new default for - #the enabled flag if one was specified. + # Preprocessor is configurable. Make sure to pass in new default for + # the enabled flag if one was specified. self.register_preprocessor(preprocessor(parent=self), enabled) elif isclass: - #Preprocessor is not configurable, construct it + # Preprocessor is not configurable, construct it self.register_preprocessor(preprocessor(), enabled) else: - #Preprocessor is an instance of something without a __call__ - #attribute. + # Preprocessor is an instance of something without a __call__ + # attribute. raise TypeError('preprocessor') - def register_filter(self, name, jinja_filter): - """ - Register a filter. - A filter is a function that accepts and acts on one string. - The filters are accesible within the Jinja templating engine. - - Parameters - ---------- - name : str - name to give the filter in the Jinja engine - filter : filter - """ - if jinja_filter is None: - raise TypeError('filter') - isclass = isinstance(jinja_filter, type) - constructed = not isclass - - #Handle filter's registration based on it's type - if constructed and isinstance(jinja_filter, py3compat.string_types): - #filter is a string, import the namespace and recursively call - #this register_filter method - filter_cls = import_item(jinja_filter) - return self.register_filter(name, filter_cls) - - if constructed and hasattr(jinja_filter, '__call__'): - #filter is a function, no need to construct it. - self.environment.filters[name] = jinja_filter - return jinja_filter - - elif isclass and isinstance(jinja_filter, MetaHasTraits): - #filter is configurable. Make sure to pass in new default for - #the enabled flag if one was specified. - filter_instance = jinja_filter(parent=self) - self.register_filter(name, filter_instance ) - - elif isclass: - #filter is not configurable, construct it - filter_instance = jinja_filter() - self.register_filter(name, filter_instance) - - else: - #filter is an instance of something without a __call__ - #attribute. - raise TypeError('filter') - - - def _init_template(self): - """ - Make sure a template name is specified. If one isn't specified, try to - build one from the information we know. - """ - self._template_file_changed('template_file', self.template_file, self.template_file) - - - def _init_environment(self, extra_loaders=None): - """ - Create the Jinja templating environment. - """ - here = os.path.dirname(os.path.realpath(__file__)) - loaders = [] - if extra_loaders: - loaders.extend(extra_loaders) - - paths = self.template_path - paths.extend([os.path.join(here, self.default_template_path), - os.path.join(here, self.template_skeleton_path)]) - loaders.append(FileSystemLoader(paths)) - - self.environment = Environment( - loader= ChoiceLoader(loaders), - extensions=JINJA_EXTENSIONS - ) - - #Set special Jinja2 syntax that will not conflict with latex. - if self.jinja_logic_block_start: - self.environment.block_start_string = self.jinja_logic_block_start - if self.jinja_logic_block_end: - self.environment.block_end_string = self.jinja_logic_block_end - if self.jinja_variable_block_start: - self.environment.variable_start_string = self.jinja_variable_block_start - if self.jinja_variable_block_end: - self.environment.variable_end_string = self.jinja_variable_block_end - if self.jinja_comment_block_start: - self.environment.comment_start_string = self.jinja_comment_block_start - if self.jinja_comment_block_end: - self.environment.comment_end_string = self.jinja_comment_block_end - - def _init_preprocessors(self): """ Register all of the preprocessors needed for this exporter, disabled unless specified explicitly. """ - self._preprocessors = [] + if self._preprocessors is None: + self._preprocessors = [] #Load default preprocessors (not necessarly enabled by default). if self.default_preprocessors: @@ -451,21 +228,6 @@ class Exporter(LoggingConfigurable): if self.preprocessors: for preprocessor in self.preprocessors: self.register_preprocessor(preprocessor, enabled=True) - - - def _init_filters(self): - """ - Register all of the filters required for the exporter. - """ - - #Add default filters to the Jinja2 environment - for key, value in default_filters.items(): - self.register_filter(key, value) - - #Load user filters. Overwrite existing filters if need be. - if self.filters: - for key, user_filter in self.filters.items(): - self.register_filter(key, user_filter) def _init_resources(self, resources): @@ -476,7 +238,7 @@ class Exporter(LoggingConfigurable): if not isinstance(resources, ResourcesDict): new_resources = ResourcesDict() new_resources.update(resources) - resources = new_resources + resources = new_resources #Make sure the metadata extension exists in resources if 'metadata' in resources: @@ -484,29 +246,28 @@ class Exporter(LoggingConfigurable): resources['metadata'] = ResourcesDict(resources['metadata']) else: resources['metadata'] = ResourcesDict() - if not resources['metadata']['name']: + if not resources['metadata']['name']: resources['metadata']['name'] = 'Notebook' #Set the output extension resources['output_extension'] = self.file_extension return resources - + def _preprocess(self, nb, resources): """ Preprocess the notebook before passing it into the Jinja engine. - To preprocess the notebook is to apply all of the - + To preprocess the notebook is to apply all of the + Parameters ---------- nb : notebook node notebook that is being exported. resources : a dict of additional resources that can be accessed read/write by preprocessors - and filters. """ - - # Do a copy.deepcopy first, + + # Do a copy.deepcopy first, # we are never safe enough with what the preprocessors could do. nbc = copy.deepcopy(nb) resc = copy.deepcopy(resources) diff --git a/IPython/nbconvert/exporters/html.py b/IPython/nbconvert/exporters/html.py index 6e1b18f..f65b65b 100644 --- a/IPython/nbconvert/exporters/html.py +++ b/IPython/nbconvert/exporters/html.py @@ -19,13 +19,13 @@ from IPython.utils.traitlets import Unicode, List from IPython.nbconvert import preprocessors from IPython.config import Config -from .exporter import Exporter +from .templateexporter import TemplateExporter #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- -class HTMLExporter(Exporter): +class HTMLExporter(TemplateExporter): """ Exports a basic HTML document. This exporter assists with the export of HTML. Inherit from it if you are writing your own HTML template and need @@ -46,7 +46,10 @@ class HTMLExporter(Exporter): c = Config({ 'CSSHTMLHeaderPreprocessor':{ 'enabled':True - } + }, + 'HighlightMagicsPreprocessor': { + 'enabled':True + } }) c.merge(super(HTMLExporter,self).default_config) return c diff --git a/IPython/nbconvert/exporters/latex.py b/IPython/nbconvert/exporters/latex.py old mode 100755 new mode 100644 index 339c6d4..6d40c5f --- a/IPython/nbconvert/exporters/latex.py +++ b/IPython/nbconvert/exporters/latex.py @@ -24,13 +24,13 @@ from IPython.utils.traitlets import Unicode, List from IPython.config import Config from IPython.nbconvert import filters, preprocessors -from .exporter import Exporter +from .templateexporter import TemplateExporter #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- -class LatexExporter(Exporter): +class LatexExporter(TemplateExporter): """ Exports to a Latex template. Inherit from this class if your template is LaTeX based and you need custom tranformers/filters. Inherit from it if @@ -45,7 +45,7 @@ class LatexExporter(Exporter): help="Extension of the file that should be written to disk") default_template = Unicode('article', config=True, help="""Template of the - data format to use. I.E. 'full' or 'basic'""") + data format to use. I.E. 'article' or 'report'""") #Latex constants default_template_path = Unicode( @@ -85,6 +85,9 @@ class LatexExporter(Exporter): }, 'SphinxPreprocessor': { 'enabled':True + }, + 'HighlightMagicsPreprocessor': { + 'enabled':True } }) c.merge(super(LatexExporter,self).default_config) diff --git a/IPython/nbconvert/exporters/markdown.py b/IPython/nbconvert/exporters/markdown.py index 160fd67..b556542 100644 --- a/IPython/nbconvert/exporters/markdown.py +++ b/IPython/nbconvert/exporters/markdown.py @@ -13,15 +13,16 @@ Exporter that will export your ipynb to Markdown. # Imports #----------------------------------------------------------------------------- +from IPython.config import Config from IPython.utils.traitlets import Unicode -from .exporter import Exporter +from .templateexporter import TemplateExporter #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- -class MarkdownExporter(Exporter): +class MarkdownExporter(TemplateExporter): """ Exports to a markdown document (.md) """ @@ -29,3 +30,9 @@ class MarkdownExporter(Exporter): file_extension = Unicode( 'md', config=True, help="Extension of the file that should be written to disk") + + @property + def default_config(self): + c = Config({'ExtractOutputPreprocessor':{'enabled':True}}) + c.merge(super(MarkdownExporter,self).default_config) + return c diff --git a/IPython/nbconvert/exporters/python.py b/IPython/nbconvert/exporters/python.py index 9fe2a94..a3b4751 100644 --- a/IPython/nbconvert/exporters/python.py +++ b/IPython/nbconvert/exporters/python.py @@ -15,13 +15,13 @@ Python exporter which exports Notebook code into a PY file. from IPython.utils.traitlets import Unicode -from .exporter import Exporter +from .templateexporter import TemplateExporter #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- -class PythonExporter(Exporter): +class PythonExporter(TemplateExporter): """ Exports a Python code file. """ diff --git a/IPython/nbconvert/exporters/rst.py b/IPython/nbconvert/exporters/rst.py index d22c6af..e9ac174 100644 --- a/IPython/nbconvert/exporters/rst.py +++ b/IPython/nbconvert/exporters/rst.py @@ -16,13 +16,13 @@ Exporter for exporting notebooks to restructured text. from IPython.utils.traitlets import Unicode from IPython.config import Config -from .exporter import Exporter +from .templateexporter import TemplateExporter #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- -class RSTExporter(Exporter): +class RSTExporter(TemplateExporter): """ Exports restructured text documents. """ diff --git a/IPython/nbconvert/exporters/slides.py b/IPython/nbconvert/exporters/slides.py index 9920206..39b083c 100644 --- a/IPython/nbconvert/exporters/slides.py +++ b/IPython/nbconvert/exporters/slides.py @@ -19,13 +19,13 @@ from IPython.utils.traitlets import Unicode from IPython.nbconvert import preprocessors from IPython.config import Config -from .exporter import Exporter +from .templateexporter import TemplateExporter #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- -class SlidesExporter(Exporter): +class SlidesExporter(TemplateExporter): """ Exports slides """ @@ -47,6 +47,9 @@ class SlidesExporter(Exporter): 'RevealHelpPreprocessor':{ 'enabled':True, }, + 'HighlightMagicsPreprocessor': { + 'enabled':True + } }) c.merge(super(SlidesExporter,self).default_config) return c diff --git a/IPython/nbconvert/exporters/templateexporter.py b/IPython/nbconvert/exporters/templateexporter.py new file mode 100644 index 0000000..d550901 --- /dev/null +++ b/IPython/nbconvert/exporters/templateexporter.py @@ -0,0 +1,316 @@ +"""This module defines Exporter, a highly configurable converter +that uses Jinja2 to export notebook files into different formats. +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from __future__ import print_function, absolute_import + +# Stdlib imports +import os + +# other libs/dependencies +from jinja2 import Environment, FileSystemLoader, ChoiceLoader, TemplateNotFound + +# IPython imports +from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Dict, Any +from IPython.utils.importstring import import_item +from IPython.utils import py3compat, text + +from IPython.nbconvert import filters +from .exporter import Exporter + +#----------------------------------------------------------------------------- +# Globals and constants +#----------------------------------------------------------------------------- + +#Jinja2 extensions to load. +JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] + +default_filters = { + 'indent': text.indent, + 'markdown2html': filters.markdown2html, + 'ansi2html': filters.ansi2html, + 'filter_data_type': filters.DataTypeFilter, + 'get_lines': filters.get_lines, + 'highlight2html': filters.highlight2html, + 'highlight2latex': filters.highlight2latex, + 'ipython2python': filters.ipython2python, + 'posix_path': filters.posix_path, + 'markdown2latex': filters.markdown2latex, + 'markdown2rst': filters.markdown2rst, + 'comment_lines': filters.comment_lines, + 'strip_ansi': filters.strip_ansi, + 'strip_dollars': filters.strip_dollars, + 'strip_files_prefix': filters.strip_files_prefix, + 'html2text' : filters.html2text, + 'add_anchor': filters.add_anchor, + 'ansi2latex': filters.ansi2latex, + 'strip_math_space': filters.strip_math_space, + 'wrap_text': filters.wrap_text, + 'escape_latex': filters.escape_latex, + 'citation2latex': filters.citation2latex, + 'path2url': filters.path2url, + 'add_prompts': filters.add_prompts, +} + +#----------------------------------------------------------------------------- +# Class +#----------------------------------------------------------------------------- + +class TemplateExporter(Exporter): + """ + Exports notebooks into other file formats. Uses Jinja 2 templating engine + to output new formats. Inherit from this class if you are creating a new + template type along with new filters/preprocessors. If the filters/ + preprocessors provided by default suffice, there is no need to inherit from + this class. Instead, override the template_file and file_extension + traits via a config file. + + {filters} + """ + + # finish the docstring + __doc__ = __doc__.format(filters = '- '+'\n - '.join(default_filters.keys())) + + + template_file = Unicode(u'default', + config=True, + help="Name of the template file to use") + def _template_file_changed(self, name, old, new): + if new == 'default': + self.template_file = self.default_template + else: + self.template_file = new + self.template = None + self._load_template() + + default_template = Unicode(u'') + template = Any() + environment = Any() + + template_path = List(['.'], config=True) + def _template_path_changed(self, name, old, new): + self._load_template() + + default_template_path = Unicode( + os.path.join("..", "templates"), + help="Path where the template files are located.") + + template_skeleton_path = Unicode( + os.path.join("..", "templates", "skeleton"), + help="Path where the template skeleton files are located.") + + #Jinja block definitions + jinja_comment_block_start = Unicode("", config=True) + jinja_comment_block_end = Unicode("", config=True) + jinja_variable_block_start = Unicode("", config=True) + jinja_variable_block_end = Unicode("", config=True) + jinja_logic_block_start = Unicode("", config=True) + jinja_logic_block_end = Unicode("", config=True) + + #Extension that the template files use. + template_extension = Unicode(".tpl", config=True) + + filters = Dict(config=True, + help="""Dictionary of filters, by name and namespace, to add to the Jinja + environment.""") + + + def __init__(self, config=None, extra_loaders=None, **kw): + """ + Public constructor + + Parameters + ---------- + config : config + User configuration instance. + extra_loaders : list[of Jinja Loaders] + ordered list of Jinja loader to find templates. Will be tried in order + before the default FileSystem ones. + template : str (optional, kw arg) + Template to use when exporting. + """ + if not config: + config = self.default_config + + super(Exporter, self).__init__(config=config, **kw) + + #Init + self._init_template() + self._init_environment(extra_loaders=extra_loaders) + self._init_preprocessors() + self._init_filters() + + + def _load_template(self): + """Load the Jinja template object from the template file + + This is a no-op if the template attribute is already defined, + or the Jinja environment is not setup yet. + + This is triggered by various trait changes that would change the template. + """ + if self.template is not None: + return + # called too early, do nothing + if self.environment is None: + return + # Try different template names during conversion. First try to load the + # template by name with extension added, then try loading the template + # as if the name is explicitly specified, then try the name as a + # 'flavor', and lastly just try to load the template by module name. + module_name = self.__module__.rsplit('.', 1)[-1] + try_names = [] + if self.template_file: + try_names.extend([ + self.template_file + self.template_extension, + self.template_file, + module_name + '_' + self.template_file + self.template_extension, + ]) + try_names.append(module_name + self.template_extension) + for try_name in try_names: + self.log.debug("Attempting to load template %s", try_name) + try: + self.template = self.environment.get_template(try_name) + except (TemplateNotFound, IOError): + pass + except Exception as e: + self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True) + else: + self.log.info("Loaded template %s", try_name) + break + + def from_notebook_node(self, nb, resources=None, **kw): + """ + Convert a notebook from a notebook node instance. + + Parameters + ---------- + nb : Notebook node + resources : dict (**kw) + of additional resources that can be accessed read/write by + preprocessors and filters. + """ + nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw) + + self._load_template() + + if self.template is not None: + output = self.template.render(nb=nb_copy, resources=resources) + else: + raise IOError('template file "%s" could not be found' % self.template_file) + return output, resources + + + def register_filter(self, name, jinja_filter): + """ + Register a filter. + A filter is a function that accepts and acts on one string. + The filters are accesible within the Jinja templating engine. + + Parameters + ---------- + name : str + name to give the filter in the Jinja engine + filter : filter + """ + if jinja_filter is None: + raise TypeError('filter') + isclass = isinstance(jinja_filter, type) + constructed = not isclass + + #Handle filter's registration based on it's type + if constructed and isinstance(jinja_filter, py3compat.string_types): + #filter is a string, import the namespace and recursively call + #this register_filter method + filter_cls = import_item(jinja_filter) + return self.register_filter(name, filter_cls) + + if constructed and hasattr(jinja_filter, '__call__'): + #filter is a function, no need to construct it. + self.environment.filters[name] = jinja_filter + return jinja_filter + + elif isclass and isinstance(jinja_filter, MetaHasTraits): + #filter is configurable. Make sure to pass in new default for + #the enabled flag if one was specified. + filter_instance = jinja_filter(parent=self) + self.register_filter(name, filter_instance ) + + elif isclass: + #filter is not configurable, construct it + filter_instance = jinja_filter() + self.register_filter(name, filter_instance) + + else: + #filter is an instance of something without a __call__ + #attribute. + raise TypeError('filter') + + + def _init_template(self): + """ + Make sure a template name is specified. If one isn't specified, try to + build one from the information we know. + """ + self._template_file_changed('template_file', self.template_file, self.template_file) + + + def _init_environment(self, extra_loaders=None): + """ + Create the Jinja templating environment. + """ + here = os.path.dirname(os.path.realpath(__file__)) + loaders = [] + if extra_loaders: + loaders.extend(extra_loaders) + + paths = self.template_path + paths.extend([os.path.join(here, self.default_template_path), + os.path.join(here, self.template_skeleton_path)]) + loaders.append(FileSystemLoader(paths)) + + self.environment = Environment( + loader= ChoiceLoader(loaders), + extensions=JINJA_EXTENSIONS + ) + + #Set special Jinja2 syntax that will not conflict with latex. + if self.jinja_logic_block_start: + self.environment.block_start_string = self.jinja_logic_block_start + if self.jinja_logic_block_end: + self.environment.block_end_string = self.jinja_logic_block_end + if self.jinja_variable_block_start: + self.environment.variable_start_string = self.jinja_variable_block_start + if self.jinja_variable_block_end: + self.environment.variable_end_string = self.jinja_variable_block_end + if self.jinja_comment_block_start: + self.environment.comment_start_string = self.jinja_comment_block_start + if self.jinja_comment_block_end: + self.environment.comment_end_string = self.jinja_comment_block_end + + + def _init_filters(self): + """ + Register all of the filters required for the exporter. + """ + + #Add default filters to the Jinja2 environment + for key, value in default_filters.items(): + self.register_filter(key, value) + + #Load user filters. Overwrite existing filters if need be. + if self.filters: + for key, user_filter in self.filters.items(): + self.register_filter(key, user_filter) diff --git a/IPython/nbconvert/exporters/tests/base.py b/IPython/nbconvert/exporters/tests/base.py index 5501b8c..18ae21d 100644 --- a/IPython/nbconvert/exporters/tests/base.py +++ b/IPython/nbconvert/exporters/tests/base.py @@ -27,4 +27,4 @@ class ExportersTestsBase(TestsBase): def _get_notebook(self): return os.path.join(self._get_files_path(), 'notebook2.ipynb') - \ No newline at end of file + diff --git a/IPython/nbconvert/exporters/tests/test_export.py b/IPython/nbconvert/exporters/tests/test_export.py index 96e02fa..ccf7de5 100644 --- a/IPython/nbconvert/exporters/tests/test_export.py +++ b/IPython/nbconvert/exporters/tests/test_export.py @@ -99,4 +99,4 @@ class TestExport(ExportersTestsBase): (output, resources) = export(None, self._get_notebook()) except TypeError: pass - \ No newline at end of file + diff --git a/IPython/nbconvert/exporters/tests/test_exporter.py b/IPython/nbconvert/exporters/tests/test_exporter.py index edaff1c..ed3cd52 100644 --- a/IPython/nbconvert/exporters/tests/test_exporter.py +++ b/IPython/nbconvert/exporters/tests/test_exporter.py @@ -17,7 +17,7 @@ Module with tests for exporter.py from IPython.config import Config from .base import ExportersTestsBase -from .cheese import CheesePreprocessor +from ...preprocessors.base import Preprocessor from ..exporter import Exporter @@ -25,84 +25,35 @@ from ..exporter import Exporter # Class #----------------------------------------------------------------------------- +class PizzaPreprocessor(Preprocessor): + """Simple preprocessor that adds a 'pizza' entry to the NotebookNode. Used + to test Exporter. + """ + + def preprocess(self, nb, resources): + nb['pizza'] = 'cheese' + return nb, resources + + class TestExporter(ExportersTestsBase): """Contains test functions for exporter.py""" def test_constructor(self): - """ - Can an Exporter be constructed? - """ + """Can an Exporter be constructed?""" Exporter() def test_export(self): - """ - Can an Exporter export something? - """ - exporter = self._make_exporter() - (output, resources) = exporter.from_filename(self._get_notebook()) - assert len(output) > 0 - - - def test_extract_outputs(self): - """ - If the ExtractOutputPreprocessor is enabled, are outputs extracted? - """ - config = Config({'ExtractOutputPreprocessor': {'enabled': True}}) - exporter = self._make_exporter(config=config) - (output, resources) = exporter.from_filename(self._get_notebook()) - assert resources is not None - assert isinstance(resources['outputs'], dict) - assert len(resources['outputs']) > 0 - - - def test_preprocessor_class(self): - """ - Can a preprocessor be added to the preprocessors list by class type? - """ - config = Config({'Exporter': {'preprocessors': [CheesePreprocessor]}}) - exporter = self._make_exporter(config=config) - (output, resources) = exporter.from_filename(self._get_notebook()) - assert resources is not None - assert resources['cheese'] == 'real' - - - def test_preprocessor_instance(self): - """ - Can a preprocessor be added to the preprocessors list by instance? - """ - config = Config({'Exporter': {'preprocessors': [CheesePreprocessor()]}}) - exporter = self._make_exporter(config=config) - (output, resources) = exporter.from_filename(self._get_notebook()) - assert resources is not None - assert resources['cheese'] == 'real' - - - def test_preprocessor_dottedobjectname(self): - """ - Can a preprocessor be added to the preprocessors list by dotted object name? - """ - config = Config({'Exporter': {'preprocessors': ['IPython.nbconvert.exporters.tests.cheese.CheesePreprocessor']}}) - exporter = self._make_exporter(config=config) - (output, resources) = exporter.from_filename(self._get_notebook()) - assert resources is not None - assert resources['cheese'] == 'real' - - - def test_preprocessor_via_method(self): - """ - Can a preprocessor be added via the Exporter convenience method? - """ - exporter = self._make_exporter() - exporter.register_preprocessor(CheesePreprocessor, enabled=True) - (output, resources) = exporter.from_filename(self._get_notebook()) - assert resources is not None - assert resources['cheese'] == 'real' - - - def _make_exporter(self, config=None): - #Create the exporter instance, make sure to set a template name since - #the base Exporter doesn't have a template associated with it. - exporter = Exporter(config=config, template_file='python') - return exporter + """Can an Exporter export something?""" + exporter = Exporter() + (notebook, resources) = exporter.from_filename(self._get_notebook()) + assert isinstance(notebook, dict) + + + def test_preprocessor(self): + """Do preprocessors work?""" + config = Config({'Exporter': {'preprocessors': [PizzaPreprocessor()]}}) + exporter = Exporter(config=config) + (notebook, resources) = exporter.from_filename(self._get_notebook()) + self.assertEqual(notebook['pizza'], 'cheese') diff --git a/IPython/nbconvert/exporters/tests/test_latex.py b/IPython/nbconvert/exporters/tests/test_latex.py index 72c3efc..f1dc349 100644 --- a/IPython/nbconvert/exporters/tests/test_latex.py +++ b/IPython/nbconvert/exporters/tests/test_latex.py @@ -44,18 +44,18 @@ class TestLatexExporter(ExportersTestsBase): @onlyif_cmds_exist('pandoc') def test_export_book(self): """ - Can a LatexExporter export using 'book' template? + Can a LatexExporter export using 'report' template? """ - (output, resources) = LatexExporter(template_file='book').from_filename(self._get_notebook()) + (output, resources) = LatexExporter(template_file='report').from_filename(self._get_notebook()) assert len(output) > 0 @onlyif_cmds_exist('pandoc') def test_export_basic(self): """ - Can a LatexExporter export using 'basic' template? + Can a LatexExporter export using 'article' template? """ - (output, resources) = LatexExporter(template_file='basic').from_filename(self._get_notebook()) + (output, resources) = LatexExporter(template_file='article').from_filename(self._get_notebook()) assert len(output) > 0 diff --git a/IPython/nbconvert/exporters/tests/test_templateexporter.py b/IPython/nbconvert/exporters/tests/test_templateexporter.py new file mode 100644 index 0000000..03bfde3 --- /dev/null +++ b/IPython/nbconvert/exporters/tests/test_templateexporter.py @@ -0,0 +1,108 @@ +""" +Module with tests for templateexporter.py +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from IPython.config import Config + +from .base import ExportersTestsBase +from .cheese import CheesePreprocessor +from ..templateexporter import TemplateExporter + + +#----------------------------------------------------------------------------- +# Class +#----------------------------------------------------------------------------- + +class TestExporter(ExportersTestsBase): + """Contains test functions for exporter.py""" + + + def test_constructor(self): + """ + Can a TemplateExporter be constructed? + """ + TemplateExporter() + + + def test_export(self): + """ + Can a TemplateExporter export something? + """ + exporter = self._make_exporter() + (output, resources) = exporter.from_filename(self._get_notebook()) + assert len(output) > 0 + + + def test_extract_outputs(self): + """ + If the ExtractOutputPreprocessor is enabled, are outputs extracted? + """ + config = Config({'ExtractOutputPreprocessor': {'enabled': True}}) + exporter = self._make_exporter(config=config) + (output, resources) = exporter.from_filename(self._get_notebook()) + assert resources is not None + assert isinstance(resources['outputs'], dict) + assert len(resources['outputs']) > 0 + + + def test_preprocessor_class(self): + """ + Can a preprocessor be added to the preprocessors list by class type? + """ + config = Config({'Exporter': {'preprocessors': [CheesePreprocessor]}}) + exporter = self._make_exporter(config=config) + (output, resources) = exporter.from_filename(self._get_notebook()) + assert resources is not None + assert resources['cheese'] == 'real' + + + def test_preprocessor_instance(self): + """ + Can a preprocessor be added to the preprocessors list by instance? + """ + config = Config({'Exporter': {'preprocessors': [CheesePreprocessor()]}}) + exporter = self._make_exporter(config=config) + (output, resources) = exporter.from_filename(self._get_notebook()) + assert resources is not None + assert resources['cheese'] == 'real' + + + def test_preprocessor_dottedobjectname(self): + """ + Can a preprocessor be added to the preprocessors list by dotted object name? + """ + config = Config({'Exporter': {'preprocessors': ['IPython.nbconvert.exporters.tests.cheese.CheesePreprocessor']}}) + exporter = self._make_exporter(config=config) + (output, resources) = exporter.from_filename(self._get_notebook()) + assert resources is not None + assert resources['cheese'] == 'real' + + + def test_preprocessor_via_method(self): + """ + Can a preprocessor be added via the Exporter convenience method? + """ + exporter = self._make_exporter() + exporter.register_preprocessor(CheesePreprocessor, enabled=True) + (output, resources) = exporter.from_filename(self._get_notebook()) + assert resources is not None + assert resources['cheese'] == 'real' + + + def _make_exporter(self, config=None): + # Create the exporter instance, make sure to set a template name since + # the base TemplateExporter doesn't have a template associated with it. + exporter = TemplateExporter(config=config, template_file='python') + return exporter diff --git a/IPython/nbconvert/filters/__init__.py b/IPython/nbconvert/filters/__init__.py index 9751c94..1acf916 100755 --- a/IPython/nbconvert/filters/__init__.py +++ b/IPython/nbconvert/filters/__init__.py @@ -1,6 +1,7 @@ from .ansi import * +from .citation import * from .datatypefilter import * from .highlight import * from .latex import * from .markdown import * -from .strings import * \ No newline at end of file +from .strings import * diff --git a/IPython/nbconvert/filters/citation.py b/IPython/nbconvert/filters/citation.py new file mode 100644 index 0000000..1442d55 --- /dev/null +++ b/IPython/nbconvert/filters/citation.py @@ -0,0 +1,72 @@ +"""Citation handling for LaTeX output.""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Code +#----------------------------------------------------------------------------- + + +__all__ = ['citation2latex'] + + +def citation2latex(s): + """Parse citations in Markdown cells. + + This looks for HTML tags having a data attribute names `data-cite` + and replaces it by the call to LaTeX cite command. The tranformation + looks like this: + + `(Granger, 2013)` + + Becomes + + `\\cite{granger}` + + Any HTML tag can be used, which allows the citations to be formatted + in HTML in any manner. + """ + try: + from lxml import html + except ImportError: + return s + + tree = html.fragment_fromstring(s, create_parent='div') + _process_node_cite(tree) + s = html.tostring(tree, encoding='unicode') + if s.endswith('
'): + s = s[:-6] + if s.startswith('
'): + s = s[5:] + return s + + +def _process_node_cite(node): + """Do the citation replacement as we walk the lxml tree.""" + + def _get(o, name): + value = getattr(o, name, None) + return '' if value is None else value + + if 'data-cite' in node.attrib: + cite = '\cite{%(ref)s}' % {'ref': node.attrib['data-cite']} + prev = node.getprevious() + if prev is not None: + prev.tail = _get(prev, 'tail') + cite + _get(node, 'tail') + else: + parent = node.getparent() + if parent is not None: + parent.text = _get(parent, 'text') + cite + _get(node, 'tail') + try: + node.getparent().remove(node) + except AttributeError: + pass + else: + for child in node: + _process_node_cite(child) diff --git a/IPython/nbconvert/filters/highlight.py b/IPython/nbconvert/filters/highlight.py index afb05b5..b390e6c 100644 --- a/IPython/nbconvert/filters/highlight.py +++ b/IPython/nbconvert/filters/highlight.py @@ -14,7 +14,7 @@ from within Jinja templates. # Imports #----------------------------------------------------------------------------- -from pygments import highlight as pygements_highlight +from pygments import highlight as pygements_highlight from pygments.lexers import get_lexer_by_name from pygments.formatters import HtmlFormatter from pygments.formatters import LatexFormatter @@ -38,51 +38,73 @@ __all__ = [ ] -def highlight2html(source, language='ipython'): +def highlight2html(source, language='ipython', metadata=None): """ Return a syntax-highlighted version of the input source as html output. - + Parameters ---------- source : str - Source code to highlight the syntax of. + source of the cell to highlight language : str - Language to highlight the syntax of. + language to highlight the syntax of + metadata : NotebookNode cell metadata + metadata of the cell to highlight """ - - return _pygment_highlight(source, HtmlFormatter(), language) + + return _pygment_highlight(source, HtmlFormatter(), language, metadata) -def highlight2latex(source, language='ipython'): +def highlight2latex(source, language='ipython', metadata=None, strip_verbatim=False): """ Return a syntax-highlighted version of the input source as latex output. - + Parameters ---------- source : str - Source code to highlight the syntax of. + source of the cell to highlight language : str - Language to highlight the syntax of. + language to highlight the syntax of + metadata : NotebookNode cell metadata + metadata of the cell to highlight + strip_verbatim : bool + remove the Verbatim environment that pygments provides by default """ - return _pygment_highlight(source, LatexFormatter(), language) + latex = _pygment_highlight(source, LatexFormatter(), language, metadata) + if strip_verbatim: + latex = latex.replace(r'\begin{Verbatim}[commandchars=\\\{\}]' + '\n', '') + return latex.replace('\n\\end{Verbatim}\n', '') + else: + return latex + -def _pygment_highlight(source, output_formatter, language='ipython'): +def _pygment_highlight(source, output_formatter, language='ipython', metadata=None): """ Return a syntax-highlighted version of the input source - + Parameters ---------- source : str - Source code to highlight the syntax of. + source of the cell to highlight output_formatter : Pygments formatter language : str - Language to highlight the syntax of. + language to highlight the syntax of + metadata : NotebookNode cell metadata + metadata of the cell to highlight """ - + + # If the cell uses a magic extension language, + # use the magic language instead. + if language == 'ipython' \ + and metadata \ + and 'magics_language' in metadata: + + language = metadata['magics_language'] + if language == 'ipython': lexer = IPythonLexer() else: lexer = get_lexer_by_name(language, stripall=True) - return pygements_highlight(source, lexer, output_formatter) + return pygements_highlight(source, lexer, output_formatter) diff --git a/IPython/nbconvert/filters/strings.py b/IPython/nbconvert/filters/strings.py index 4b8d78f..ecd0d54 100755 --- a/IPython/nbconvert/filters/strings.py +++ b/IPython/nbconvert/filters/strings.py @@ -19,6 +19,7 @@ templates. import os import re import textwrap +from urllib2 import quote from xml.etree import ElementTree from IPython.core.interactiveshell import InteractiveShell @@ -38,6 +39,8 @@ __all__ = [ 'get_lines', 'ipython2python', 'posix_path', + 'path2url', + 'add_prompts' ] @@ -80,7 +83,7 @@ def add_anchor(html): For use in heading cells """ - h = ElementTree.fromstring(py3compat.cast_bytes_py2(html)) + h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding='utf-8')) link = html2text(h).replace(' ', '-') h.set('id', link) a = ElementTree.Element("a", {"class" : "anchor-link", "href" : "#" + link}) @@ -93,6 +96,16 @@ def add_anchor(html): return py3compat.decode(ElementTree.tostring(h), 'utf-8') +def add_prompts(code, first='>>> ', cont='... '): + """Add prompts to code snippets""" + new_code = [] + code_list = code.split('\n') + new_code.append(first + code_list[0]) + for line in code_list[1:]: + new_code.append(cont + line) + return '\n'.join(new_code) + + def strip_dollars(text): """ Remove all dollar symbols from text @@ -181,3 +194,8 @@ def posix_path(path): if os.path.sep != '/': return path.replace(os.path.sep, '/') return path + +def path2url(path): + """Turn a file path into a URL""" + parts = path.split(os.path.sep) + return '/'.join(quote(part) for part in parts) diff --git a/IPython/nbconvert/filters/tests/test_ansi.py b/IPython/nbconvert/filters/tests/test_ansi.py index c6d9d17..e24c10c 100644 --- a/IPython/nbconvert/filters/tests/test_ansi.py +++ b/IPython/nbconvert/filters/tests/test_ansi.py @@ -39,7 +39,7 @@ class TestAnsi(TestsBase): 'hello' : 'hello'} for inval, outval in correct_outputs.items(): - yield self._try_strip_ansi(inval, outval) + self._try_strip_ansi(inval, outval) def _try_strip_ansi(self, inval, outval): @@ -58,7 +58,7 @@ class TestAnsi(TestsBase): 'hello' : 'hello'} for inval, outval in correct_outputs.items(): - yield self._try_ansi2html(inval, outval) + self._try_ansi2html(inval, outval) def _try_ansi2html(self, inval, outval): @@ -77,7 +77,7 @@ class TestAnsi(TestsBase): 'hello' : 'hello'} for inval, outval in correct_outputs.items(): - yield self._try_ansi2latex(inval, outval) + self._try_ansi2latex(inval, outval) def _try_ansi2latex(self, inval, outval): diff --git a/IPython/nbconvert/filters/tests/test_citation.py b/IPython/nbconvert/filters/tests/test_citation.py new file mode 100644 index 0000000..7f068c2 --- /dev/null +++ b/IPython/nbconvert/filters/tests/test_citation.py @@ -0,0 +1,58 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from ..citation import citation2latex + +#----------------------------------------------------------------------------- +# Tests +#----------------------------------------------------------------------------- + +test_md = """ +# My Heading + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus ac magna non augue +porttitor scelerisque ac id diam Granger. Mauris elit +velit, lobortis sed interdum at, vestibulum vitae libero Perez. +Lorem ipsum dolor sit amet, consectetur adipiscing elit +Thomas. Quisque iaculis ligula ut ipsum mattis viverra. + +

Here is a plain paragraph that should be unaffected.

+ +* One Jonathan. +* Two Matthias. +* Three Paul. +""" + +test_md_parsed = """ +# My Heading + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus ac magna non augue +porttitor scelerisque ac id diam \cite{granger}. Mauris elit +velit, lobortis sed interdum at, vestibulum vitae libero \cite{fperez}. +Lorem ipsum dolor sit amet, consectetur adipiscing elit +\cite{takluyver}. Quisque iaculis ligula ut ipsum mattis viverra. + +

Here is a plain paragraph that should be unaffected.

+ +* One \cite{jdfreder}. +* Two \cite{carreau}. +* Three \cite{ivanov}. +""" + +def test_citation2latex(): + """Are citations parsed properly?""" + try: + import lxml + except ImportError: + assert test_md == citation2latex(test_md) + else: + assert test_md_parsed == citation2latex(test_md) diff --git a/IPython/nbconvert/filters/tests/test_highlight.py b/IPython/nbconvert/filters/tests/test_highlight.py index 04f9107..df25a42 100644 --- a/IPython/nbconvert/filters/tests/test_highlight.py +++ b/IPython/nbconvert/filters/tests/test_highlight.py @@ -49,13 +49,13 @@ class TestHighlight(TestsBase): def test_highlight2html(self): """highlight2html test""" for index, test in enumerate(self.tests): - yield self._try_highlight(highlight2html, test, self.tokens[index]) + self._try_highlight(highlight2html, test, self.tokens[index]) def test_highlight2latex(self): """highlight2latex test""" for index, test in enumerate(self.tests): - yield self._try_highlight(highlight2latex, test, self.tokens[index]) + self._try_highlight(highlight2latex, test, self.tokens[index]) def _try_highlight(self, method, test, tokens): diff --git a/IPython/nbconvert/filters/tests/test_latex.py b/IPython/nbconvert/filters/tests/test_latex.py index ae2914c..282e3c9 100644 --- a/IPython/nbconvert/filters/tests/test_latex.py +++ b/IPython/nbconvert/filters/tests/test_latex.py @@ -35,7 +35,7 @@ class TestLatex(TestsBase): ('','')] for test in tests: - yield self._try_escape_latex(test[0], test[1]) + self._try_escape_latex(test[0], test[1]) def _try_escape_latex(self, test, result): @@ -56,7 +56,7 @@ class TestLatex(TestsBase): ('','')] for test in tests: - yield self._try_strip_math_space(test[0], test[1]) + self._try_strip_math_space(test[0], test[1]) def _try_strip_math_space(self, test, result): diff --git a/IPython/nbconvert/filters/tests/test_markdown.py b/IPython/nbconvert/filters/tests/test_markdown.py index d0b5b3c..54e100a 100644 --- a/IPython/nbconvert/filters/tests/test_markdown.py +++ b/IPython/nbconvert/filters/tests/test_markdown.py @@ -61,14 +61,14 @@ class TestMarkdown(TestsBase): def test_markdown2latex(self): """markdown2latex test""" for index, test in enumerate(self.tests): - yield self._try_markdown(markdown2latex, test, self.tokens[index]) + self._try_markdown(markdown2latex, test, self.tokens[index]) @dec.onlyif_cmds_exist('pandoc') def test_markdown2html(self): """markdown2html test""" for index, test in enumerate(self.tests): - yield self._try_markdown(markdown2html, test, self.tokens[index]) + self._try_markdown(markdown2html, test, self.tokens[index]) @dec.onlyif_cmds_exist('pandoc') @@ -81,7 +81,7 @@ class TestMarkdown(TestsBase): tokens[1] = r'\*\*test' for index, test in enumerate(self.tests): - yield self._try_markdown(markdown2rst, test, tokens[index]) + self._try_markdown(markdown2rst, test, tokens[index]) def _try_markdown(self, method, test, tokens): diff --git a/IPython/nbconvert/filters/tests/test_strings.py b/IPython/nbconvert/filters/tests/test_strings.py index f0739c3..9fc6d63 100644 --- a/IPython/nbconvert/filters/tests/test_strings.py +++ b/IPython/nbconvert/filters/tests/test_strings.py @@ -15,10 +15,10 @@ Module with tests for Strings #----------------------------------------------------------------------------- import os -from IPython.testing import decorators as dec from ...tests.base import TestsBase from ..strings import (wrap_text, html2text, add_anchor, strip_dollars, strip_files_prefix, get_lines, comment_lines, ipython2python, posix_path, + add_prompts ) @@ -36,7 +36,7 @@ class TestStrings(TestsBase): As if the strings were thine, shouldst know of this. """ for length in [30,5,1]: - yield self._confirm_wrap_text(test_text, length) + self._confirm_wrap_text(test_text, length) def _confirm_wrap_text(self, text, length): @@ -73,7 +73,7 @@ class TestStrings(TestsBase): ('Hello', 'Hello'), ('W$o$rld', 'W$o$rld')] for test in tests: - yield self._try_strip_dollars(test[0], test[1]) + self._try_strip_dollars(test[0], test[1]) def _try_strip_dollars(self, test, result): @@ -89,7 +89,7 @@ class TestStrings(TestsBase): ('My files are in `files/`', 'My files are in `files/`'), ('files/test.html', 'files/test.html')] for test in tests: - yield self._try_files_prefix(test[0], test[1]) + self._try_files_prefix(test[0], test[1]) def _try_files_prefix(self, test, result): @@ -121,8 +121,15 @@ class TestStrings(TestsBase): ignore_spaces=True, ignore_newlines=True) def test_posix_path(self): + """posix_path test""" path_list = ['foo', 'bar'] expected = '/'.join(path_list) native = os.path.join(*path_list) filtered = posix_path(native) self.assertEqual(filtered, expected) + + def test_add_prompts(self): + """add_prompts test""" + text1 = """for i in range(10):\n i += 1\n print i""" + text2 = """>>> for i in range(10):\n... i += 1\n... print i""" + self.assertEqual(text2, add_prompts(text1)) diff --git a/IPython/nbconvert/nbconvertapp.py b/IPython/nbconvert/nbconvertapp.py index 3ea762b..8b09002 100755 --- a/IPython/nbconvert/nbconvertapp.py +++ b/IPython/nbconvert/nbconvertapp.py @@ -25,6 +25,7 @@ import glob # From IPython from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags +from IPython.core.profiledir import ProfileDir from IPython.config import catch_config_error, Configurable from IPython.utils.traitlets import ( Unicode, List, Instance, DottedObjectName, Type, CaselessStrEnum, @@ -58,12 +59,11 @@ nbconvert_aliases = {} nbconvert_aliases.update(base_aliases) nbconvert_aliases.update({ 'to' : 'NbConvertApp.export_format', - 'template' : 'Exporter.template_file', + 'template' : 'TemplateExporter.template_file', 'writer' : 'NbConvertApp.writer_class', 'post': 'NbConvertApp.postprocessor_class', 'output': 'NbConvertApp.output_base', - 'offline-slides': 'RevealHelpTransformer.url_prefix', - 'slide-notes': 'RevealHelpTransformer.speaker_notes' + 'reveal-prefix': 'RevealHelpPreprocessor.url_prefix', }) nbconvert_flags = {} @@ -87,12 +87,13 @@ class NbConvertApp(BaseIPythonApplication): return logging.INFO def _classes_default(self): - classes = [NbConvertBase] + classes = [NbConvertBase, ProfileDir] for pkg in (exporters, preprocessors, writers): for name in dir(pkg): cls = getattr(pkg, name) if isinstance(cls, type) and issubclass(cls, Configurable): classes.append(cls) + return classes description = Unicode( diff --git a/IPython/nbconvert/postprocessors/__init__.py b/IPython/nbconvert/postprocessors/__init__.py index 6cae641..9f954f3 100644 --- a/IPython/nbconvert/postprocessors/__init__.py +++ b/IPython/nbconvert/postprocessors/__init__.py @@ -1,3 +1,8 @@ from .base import PostProcessorBase from .pdf import PDFPostProcessor -from .serve import ServePostProcessor + +# protect against unavailable tornado +try: + from .serve import ServePostProcessor +except ImportError: + pass diff --git a/IPython/nbconvert/postprocessors/pdf.py b/IPython/nbconvert/postprocessors/pdf.py index 53062f3..7683ee9 100644 --- a/IPython/nbconvert/postprocessors/pdf.py +++ b/IPython/nbconvert/postprocessors/pdf.py @@ -26,35 +26,107 @@ from .base import PostProcessorBase class PDFPostProcessor(PostProcessorBase): """Writer designed to write to PDF files""" - iteration_count = Integer(3, config=True, help=""" + latex_count = Integer(3, config=True, help=""" How many times pdflatex will be called. """) - command = List(["pdflatex", "{filename}"], config=True, help=""" + latex_command = List(["pdflatex", "{filename}"], config=True, help=""" Shell command used to compile PDF.""") + bib_command = List(["bibtex", "{filename}"], config=True, help=""" + Shell command used to run bibtex.""") + verbose = Bool(False, config=True, help=""" Whether or not to display the output of the compile call. """) - def postprocess(self, input): - """ - Consume and write Jinja output a PDF. - See files.py for more... - """ - command = [c.format(filename=input) for c in self.command] - self.log.info("Building PDF: %s", command) - with open(os.devnull, 'rb') as null: - stdout = subprocess.PIPE if not self.verbose else None - for index in range(self.iteration_count): - p = subprocess.Popen(command, stdout=stdout, stdin=null) - out, err = p.communicate() - if p.returncode: - if self.verbose: - # verbose means I didn't capture stdout with PIPE, - # so it's already been displayed and `out` is None. - out = u'' - else: - out = out.decode('utf-8', 'replace') - self.log.critical(u"PDF conversion failed: %s\n%s", command, out) - return + temp_file_exts = List(['.aux', '.bbl', '.blg', '.idx', '.log', '.out'], + config=True, help=""" + Filename extensions of temp files to remove after running. + """) + + def run_command(self, command_list, filename, count, log_function): + """Run command_list count times. + + Parameters + ---------- + command_list : list + A list of args to provide to Popen. Each element of this + list will be interpolated with the filename to convert. + filename : unicode + The name of the file to convert. + count : int + How many times to run the command. + + Returns + ------- + continue : bool + A boolean indicating if the command was successful (True) + or failed (False). + """ + command = [c.format(filename=filename) for c in command_list] + times = 'time' if count == 1 else 'times' + self.log.info("Running %s %i %s: %s", command_list[0], count, times, command) + with open(os.devnull, 'rb') as null: + stdout = subprocess.PIPE if not self.verbose else None + for index in range(count): + p = subprocess.Popen(command, stdout=stdout, stdin=null) + out, err = p.communicate() + if p.returncode: + if self.verbose: + # verbose means I didn't capture stdout with PIPE, + # so it's already been displayed and `out` is None. + out = u'' + else: + out = out.decode('utf-8', 'replace') + log_function(command, out) + return False # failure + return True # success + + def run_latex(self, filename): + """Run pdflatex self.latex_count times.""" + + def log_error(command, out): + self.log.critical(u"%s failed: %s\n%s", command[0], command, out) + + return self.run_command(self.latex_command, filename, + self.latex_count, log_error) + + def run_bib(self, filename): + """Run bibtex self.latex_count times.""" + filename = os.path.splitext(filename)[0] + + def log_error(command, out): + self.log.warn('%s had problems, most likely because there were no citations', + command[0]) + self.log.debug(u"%s output: %s\n%s", command[0], command, out) + + return self.run_command(self.bib_command, filename, 1, log_error) + + def clean_temp_files(self, filename): + """Remove temporary files created by pdflatex/bibtext.""" + self.log.info("Removing temporary LaTeX files") + filename = os.path.splitext(filename)[0] + for ext in self.temp_file_exts: + try: + os.remove(filename+ext) + except OSError: + pass + + def postprocess(self, filename): + """Build a PDF by running pdflatex and bibtex""" + self.log.info("Building PDF") + cont = self.run_latex(filename) + if cont: + cont = self.run_bib(filename) + else: + self.clean_temp_files(filename) + return + if cont: + cont = self.run_latex(filename) + self.clean_temp_files(filename) + filename = os.path.splitext(filename)[0] + if os.path.isfile(filename+'.pdf'): + self.log.info('PDF successfully created') + return + diff --git a/IPython/nbconvert/postprocessors/serve.py b/IPython/nbconvert/postprocessors/serve.py index 828f5f3..8a83419 100644 --- a/IPython/nbconvert/postprocessors/serve.py +++ b/IPython/nbconvert/postprocessors/serve.py @@ -1,6 +1,4 @@ -""" -Contains postprocessor for serving nbconvert output. -""" +"""PostProcessor for serving reveal.js HTML slideshows.""" #----------------------------------------------------------------------------- #Copyright (c) 2013, the IPython Development Team. # @@ -16,40 +14,94 @@ Contains postprocessor for serving nbconvert output. import os import webbrowser -from BaseHTTPServer import HTTPServer -from SimpleHTTPServer import SimpleHTTPRequestHandler +from tornado import web, ioloop, httpserver +from tornado.httpclient import AsyncHTTPClient -from IPython.utils.traitlets import Bool +from IPython.utils.traitlets import Bool, Unicode, Int from .base import PostProcessorBase #----------------------------------------------------------------------------- # Classes #----------------------------------------------------------------------------- + +class ProxyHandler(web.RequestHandler): + """handler the proxies requests from a local prefix to a CDN""" + @web.asynchronous + def get(self, prefix, url): + """proxy a request to a CDN""" + proxy_url = "/".join([self.settings['cdn'], url]) + client = self.settings['client'] + client.fetch(proxy_url, callback=self.finish_get) + + def finish_get(self, response): + """finish the request""" + # copy potentially relevant headers + for header in ["Content-Type", "Cache-Control", "Date", "Last-Modified", "Expires"]: + if header in response.headers: + self.set_header(header, response.headers[header]) + self.finish(response.body) + class ServePostProcessor(PostProcessorBase): - """Post processor designed to serve files""" + """Post processor designed to serve files + + Proxies reveal.js requests to a CDN if no local reveal.js is present + """ open_in_browser = Bool(True, config=True, - help="""Set to False to deactivate - the opening of the browser""") + help="""Should the browser be opened automatically?""" + ) + reveal_cdn = Unicode("https://cdn.jsdelivr.net/reveal.js/2.4.0", config=True, + help="""URL for reveal.js CDN.""" + ) + reveal_prefix = Unicode("reveal.js", config=True, help="URL prefix for reveal.js") + ip = Unicode("127.0.0.1", config=True, help="The IP address to listen on.") + port = Int(8000, config=True, help="port for the server to listen on.") def postprocess(self, input): - """ - Simple implementation to serve the build directory. - """ - + """Serve the build directory with a webserver.""" + dirname, filename = os.path.split(input) + handlers = [ + (r"/(.+)", web.StaticFileHandler, {'path' : dirname}), + (r"/", web.RedirectHandler, {"url": "/%s" % filename}) + ] + + if ('://' in self.reveal_prefix or self.reveal_prefix.startswith("//")): + # reveal specifically from CDN, nothing to do + pass + elif os.path.isdir(os.path.join(dirname, self.reveal_prefix)): + # reveal prefix exists + self.log.info("Serving local %s", self.reveal_prefix) + else: + self.log.info("Redirecting %s requests to %s", self.reveal_prefix, self.reveal_cdn) + handlers.insert(0, (r"/(%s)/(.*)" % self.reveal_prefix, ProxyHandler)) + + app = web.Application(handlers, + cdn=self.reveal_cdn, + client=AsyncHTTPClient(), + ) + # hook up tornado logging to our logger + from tornado import log + log.app_log = self.log + + http_server = httpserver.HTTPServer(app) + http_server.listen(self.port, address=self.ip) + url = "http://%s:%i/%s" % (self.ip, self.port, filename) + print("Serving your slides at %s" % url) + print("Use Control-C to stop this server") + if self.open_in_browser: + webbrowser.open(url, new=2) try: - dirname, filename = os.path.split(input) - if dirname: - os.chdir(dirname) - httpd = HTTPServer(('127.0.0.1', 8000), SimpleHTTPRequestHandler) - sa = httpd.socket.getsockname() - url = "http://" + sa[0] + ":" + str(sa[1]) + "/" + filename - if self.open_in_browser: - webbrowser.open(url, new=2) - print("Serving your slides on " + url) - print("Use Control-C to stop this server.") - httpd.serve_forever() + ioloop.IOLoop.instance().start() except KeyboardInterrupt: - print("The server is shut down.") + print("\nInterrupted") + +def main(path): + """allow running this module to serve the slides""" + server = ServePostProcessor() + server(path) + +if __name__ == '__main__': + import sys + main(sys.argv[1]) diff --git a/IPython/nbconvert/postprocessors/tests/test_pdf.py b/IPython/nbconvert/postprocessors/tests/test_pdf.py index c163928..cc650d6 100644 --- a/IPython/nbconvert/postprocessors/tests/test_pdf.py +++ b/IPython/nbconvert/postprocessors/tests/test_pdf.py @@ -62,3 +62,7 @@ class TestPDF(TestsBase): # Check that the PDF was created. assert os.path.isfile('a.pdf') + + # Make sure that temp files are cleaned up + for ext in processor.temp_file_exts: + assert not os.path.isfile('a'+ext) diff --git a/IPython/nbconvert/preprocessors/__init__.py b/IPython/nbconvert/preprocessors/__init__.py index 4f3e490..3062c79 100755 --- a/IPython/nbconvert/preprocessors/__init__.py +++ b/IPython/nbconvert/preprocessors/__init__.py @@ -5,8 +5,8 @@ from .svg2pdf import SVG2PDFPreprocessor from .extractoutput import ExtractOutputPreprocessor from .revealhelp import RevealHelpPreprocessor from .latex import LatexPreprocessor -from .sphinx import SphinxPreprocessor from .csshtmlheader import CSSHTMLHeaderPreprocessor +from .highlightmagics import HighlightMagicsPreprocessor # decorated function Preprocessors from .coalescestreams import coalesce_streams diff --git a/IPython/nbconvert/preprocessors/coalescestreams.py b/IPython/nbconvert/preprocessors/coalescestreams.py index 5d906a8..873c559 100644 --- a/IPython/nbconvert/preprocessors/coalescestreams.py +++ b/IPython/nbconvert/preprocessors/coalescestreams.py @@ -19,7 +19,8 @@ def cell_preprocessor(function): Wrap a function to be executed on all cells of a notebook Wrapped Parameters - ---------- + ------------------ + cell : NotebookNode cell Notebook cell being processed resources : dictionary @@ -69,7 +70,7 @@ def coalesce_streams(cell, resources, index): last.text += output.text else: new_outputs.append(output) - last = output + last = output cell.outputs = new_outputs return cell, resources diff --git a/IPython/nbconvert/preprocessors/highlightmagics.py b/IPython/nbconvert/preprocessors/highlightmagics.py new file mode 100644 index 0000000..135fe4e --- /dev/null +++ b/IPython/nbconvert/preprocessors/highlightmagics.py @@ -0,0 +1,113 @@ +"""This preprocessor detect cells using a different language through +magic extensions such as `%%R` or `%%octave`. Cell's metadata is marked +so that the appropriate highlighter can be used in the `highlight` +filter. +""" + +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from __future__ import print_function, absolute_import + +import re + +# Our own imports +from .base import Preprocessor +from IPython.utils.traitlets import Dict + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- + + +class HighlightMagicsPreprocessor(Preprocessor): + """ + Detects and tags code cells that use a different languages than Python. + """ + + # list of magic language extensions and their associated pygment lexers + default_languages = Dict( + default_value={ + '%%R': 'r', + '%%bash': 'bash', + '%%cython': 'cython', + '%%javascript': 'javascript', + '%%julia': 'julia', + '%%latex': 'latex', + '%%octave': 'octave', + '%%perl': 'perl', + '%%ruby': 'ruby', + '%%sh': 'sh'}) + + # user defined language extensions + languages = Dict( + config=True, + help=("Syntax highlighting for magic's extension languages. " + "Each item associates a language magic extension such as %%R, " + "with a pygments lexer such as r.")) + + def __init__(self, config=None, **kw): + """Public constructor""" + + super(HighlightMagicsPreprocessor, self).__init__(config=config, **kw) + + # Update the default languages dict with the user configured ones + self.default_languages.update(self.languages) + + # build a regular expression to catch language extensions and choose + # an adequate pygments lexer + any_language = "|".join(self.default_languages.keys()) + self.re_magic_language = re.compile( + r'^\s*({0})\s+'.format(any_language)) + + def which_magic_language(self, source): + """ + When a cell uses another language through a magic extension, + the other language is returned. + If no language magic is detected, this function returns None. + + Parameters + ---------- + source: str + Source code of the cell to highlight + """ + + m = self.re_magic_language.match(source) + + if m: + # By construction of the re, the matched language must be in the + # languages dictionary + return self.default_languages[m.group(1)] + else: + return None + + def preprocess_cell(self, cell, resources, cell_index): + """ + Tags cells using a magic extension language + + Parameters + ---------- + cell : NotebookNode cell + Notebook cell being processed + resources : dictionary + Additional resources used in the conversion process. Allows + preprocessors to pass variables into the Jinja engine. + cell_index : int + Index of the cell being processed (see base.py) + """ + + # Only tag code cells + if hasattr(cell, "input") and cell.cell_type == "code": + magic_language = self.which_magic_language(cell.input) + if magic_language: + cell['metadata']['magics_language'] = magic_language + return cell, resources diff --git a/IPython/nbconvert/preprocessors/latex.py b/IPython/nbconvert/preprocessors/latex.py index 3cc9f0b..0cc4074 100755 --- a/IPython/nbconvert/preprocessors/latex.py +++ b/IPython/nbconvert/preprocessors/latex.py @@ -14,9 +14,12 @@ they are converted. #----------------------------------------------------------------------------- from __future__ import print_function, absolute_import +import os -# Our own imports -# Needed to override preprocessor +# Third-party import, needed for Pygments latex definitions. +from pygments.formatters import LatexFormatter + +# ipy imports from .base import (Preprocessor) from IPython.nbconvert import filters @@ -29,6 +32,24 @@ class LatexPreprocessor(Preprocessor): Converter for latex destined documents. """ + def preprocess(self, nb, resources): + """ + Preprocessing to apply on each notebook. + + Parameters + ---------- + nb : NotebookNode + Notebook being converted + resources : dictionary + Additional resources used in the conversion process. Allows + preprocessors to pass variables into the Jinja engine. + """ + # Generate Pygments definitions for Latex + resources["latex"] = {} + resources["latex"]["pygments_definitions"] = LatexFormatter().get_style_defs() + return super(LatexPreprocessor, self).preprocess(nb, resources) + + def preprocess_cell(self, cell, resources, index): """ Apply a transformation on each cell, diff --git a/IPython/nbconvert/preprocessors/revealhelp.py b/IPython/nbconvert/preprocessors/revealhelp.py index ae1110e..d9e213f 100755 --- a/IPython/nbconvert/preprocessors/revealhelp.py +++ b/IPython/nbconvert/preprocessors/revealhelp.py @@ -24,15 +24,14 @@ from IPython.utils.traitlets import Unicode, Bool class RevealHelpPreprocessor(Preprocessor): - url_prefix = Unicode('//cdn.jsdelivr.net/reveal.js/2.4.0', - config=True, - help="""If you want to use a local reveal.js library, - use 'url_prefix':'reveal.js' in your config object.""") - - speaker_notes = Bool(False, - config=True, - help="""If you want to use the speaker notes - set this to True.""") + url_prefix = Unicode('reveal.js', config=True, + help="""The URL prefix for reveal.js. + This can be a a relative URL for a local copy of reveal.js, + or point to a CDN. + + For speaker notes to work, a local reveal.js prefix must be used. + """ + ) def preprocess(self, nb, resources): """ @@ -65,30 +64,4 @@ class RevealHelpPreprocessor(Preprocessor): if not isinstance(resources['reveal'], dict): resources['reveal'] = {} resources['reveal']['url_prefix'] = self.url_prefix - resources['reveal']['notes_prefix'] = self.url_prefix - - cdn = 'http://cdn.jsdelivr.net/reveal.js/2.4.0' - local = 'local' - html_path = 'plugin/notes/notes.html' - js_path = 'plugin/notes/notes.js' - - html_infile = os.path.join(cdn, html_path) - js_infile = os.path.join(cdn, js_path) - html_outfile = os.path.join(local, html_path) - js_outfile = os.path.join(local, js_path) - - if self.speaker_notes: - if 'outputs' not in resources: - resources['outputs'] = {} - resources['outputs'][html_outfile] = self.notes_helper(html_infile) - resources['outputs'][js_outfile] = self.notes_helper(js_infile) - resources['reveal']['notes_prefix'] = local - return nb, resources - - def notes_helper(self, infile): - """Helper function to get the content from an url.""" - - content = urllib2.urlopen(infile).read() - - return content diff --git a/IPython/nbconvert/preprocessors/sphinx.py b/IPython/nbconvert/preprocessors/sphinx.py deleted file mode 100755 index 6974291..0000000 --- a/IPython/nbconvert/preprocessors/sphinx.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Module that allows custom Sphinx parameters to be set on the notebook and -on the 'other' object passed into Jinja. Called prior to Jinja conversion -process. -""" -#----------------------------------------------------------------------------- -# Copyright (c) 2013, the IPython Development Team. -# -# Distributed under the terms of the Modified BSD License. -# -# The full license is in the file COPYING.txt, distributed with this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -from __future__ import print_function, absolute_import - -# Stdlib imports -import os.path - -# Used to set the default date to today's date -from datetime import date - -# Third-party imports -# Needed for Pygments latex definitions. -from pygments.formatters import LatexFormatter - -# Our own imports -# Configurable traitlets -from IPython.utils.traitlets import Unicode, Bool - -# Needed to override preprocessor -from .base import (Preprocessor) - -from IPython.nbconvert.utils import console - -#----------------------------------------------------------------------------- -# Classes and functions -#----------------------------------------------------------------------------- - -class SphinxPreprocessor(Preprocessor): - """ - Sphinx utility preprocessor. - - This preprocessor is used to set variables needed by the latex to build - Sphinx stylized templates. - """ - - interactive = Bool(False, config=True, help=""" - Allows you to define whether or not the Sphinx exporter will prompt - you for input during the conversion process. If this is set to false, - the author, version, release, date, and chapter_style traits should - be set. - """) - - author = Unicode("Unknown Author", config=True, help="Author name") - - version = Unicode("", config=True, help=""" - Version number - You can leave this blank if you do not want to render a version number. - Example: "1.0.0" - """) - - release = Unicode("", config=True, help=""" - Release name - You can leave this blank if you do not want to render a release name. - Example: "Rough Draft" - """) - - publish_date = Unicode("", config=True, help=""" - Publish date - This is the date to render on the document as the publish date. - Leave this blank to default to todays date. - Example: "June 12, 1990" - """) - - chapter_style = Unicode("Bjarne", config=True, help=""" - Sphinx chapter style - This is the style to use for the chapter headers in the document. - You may choose one of the following: - "Bjarne" (default) - "Lenny" - "Glenn" - "Conny" - "Rejne" - "Sonny" (used for international documents) - """) - - output_style = Unicode("notebook", config=True, help=""" - Nbconvert Ipython - notebook input/output formatting style. - You may choose one of the following: - "simple (recommended for long code segments)" - "notebook" (default) - """) - - center_output = Bool(False, config=True, help=""" - Optional attempt to center all output. If this is false, no additional - formatting is applied. - """) - - use_headers = Bool(True, config=True, help=""" - Whether not a header should be added to the document. - """) - - #Allow the user to override the title of the notebook (useful for - #fancy document titles that the file system doesn't support.) - overridetitle = Unicode("", config=True, help="") - - - def preprocess(self, nb, resources): - """ - Sphinx preprocessing to apply on each notebook. - - Parameters - ---------- - nb : NotebookNode - Notebook being converted - resources : dictionary - Additional resources used in the conversion process. Allows - preprocessors to pass variables into the Jinja engine. - """ - # import sphinx here, so that sphinx is not a dependency when it's not used - import sphinx - - # TODO: Add versatile method of additional notebook metadata. Include - # handling of multiple files. For now use a temporay namespace, - # '_draft' to signify that this needs to change. - if not isinstance(resources["sphinx"], dict): - resources["sphinx"] = {} - - if self.interactive: - - # Prompt the user for additional meta data that doesn't exist currently - # but would be usefull for Sphinx. - resources["sphinx"]["author"] = self._prompt_author() - resources["sphinx"]["version"] = self._prompt_version() - resources["sphinx"]["release"] = self._prompt_release() - resources["sphinx"]["date"] = self._prompt_date() - - # Prompt the user for the document style. - resources["sphinx"]["chapterstyle"] = self._prompt_chapter_title_style() - resources["sphinx"]["outputstyle"] = self._prompt_output_style() - - # Small options - resources["sphinx"]["centeroutput"] = console.prompt_boolean("Do you want to center the output? (false)", False) - resources["sphinx"]["header"] = console.prompt_boolean("Should a Sphinx document header be used? (true)", True) - else: - - # Try to use the traitlets. - resources["sphinx"]["author"] = self.author - resources["sphinx"]["version"] = self.version - resources["sphinx"]["release"] = self.release - - # Use todays date if none is provided. - if self.publish_date: - resources["sphinx"]["date"] = self.publish_date - elif len(resources['metadata']['modified_date'].strip()) == 0: - resources["sphinx"]["date"] = date.today().strftime("%B %-d, %Y") - else: - resources["sphinx"]["date"] = resources['metadata']['modified_date'] - - # Sphinx traitlets. - resources["sphinx"]["chapterstyle"] = self.chapter_style - resources["sphinx"]["outputstyle"] = self.output_style - resources["sphinx"]["centeroutput"] = self.center_output - resources["sphinx"]["header"] = self.use_headers - - # Find and pass in the path to the Sphinx dependencies. - resources["sphinx"]["texinputs"] = os.path.realpath(os.path.join(sphinx.package_dir, "texinputs")) - - # Generate Pygments definitions for Latex - resources["sphinx"]["pygment_definitions"] = self._generate_pygments_latex_def() - - if not (self.overridetitle == None or len(self.overridetitle.strip()) == 0): - resources['metadata']['name'] = self.overridetitle - - # End - return nb, resources - - - def _generate_pygments_latex_def(self): - """ - Generate the pygments latex definitions that allows pygments - to work in latex. - """ - - return LatexFormatter().get_style_defs() - - - def _prompt_author(self): - """ - Prompt the user to input an Author name - """ - return console.input("Author name: ") - - - def _prompt_version(self): - """ - prompt the user to enter a version number - """ - return console.input("Version (ie ""1.0.0""): ") - - - def _prompt_release(self): - """ - Prompt the user to input a release name - """ - - return console.input("Release Name (ie ""Rough draft""): ") - - - def _prompt_date(self, resources): - """ - Prompt the user to enter a date - """ - - if resources['metadata']['modified_date']: - default_date = resources['metadata']['modified_date'] - else: - default_date = date.today().strftime("%B %-d, %Y") - - user_date = console.input("Date (deafults to \"" + default_date + "\"): ") - if len(user_date.strip()) == 0: - user_date = default_date - return user_date - - - def _prompt_output_style(self): - """ - Prompts the user to pick an IPython output style. - """ - - # Dictionary of available output styles - styles = {1: "simple", - 2: "notebook"} - - #Append comments to the menu when displaying it to the user. - comments = {1: "(recommended for long code segments)", - 2: "(default)"} - - return console.prompt_dictionary(styles, default_style=2, menu_comments=comments) - - - def _prompt_chapter_title_style(self): - """ - Prompts the user to pick a Sphinx chapter style - """ - - # Dictionary of available Sphinx styles - styles = {1: "Bjarne", - 2: "Lenny", - 3: "Glenn", - 4: "Conny", - 5: "Rejne", - 6: "Sonny"} - - #Append comments to the menu when displaying it to the user. - comments = {1: "(default)", - 6: "(for international documents)"} - - return console.prompt_dictionary(styles, menu_comments=comments) - diff --git a/IPython/nbconvert/preprocessors/tests/base.py b/IPython/nbconvert/preprocessors/tests/base.py index 8c72e51..d5c1459 100644 --- a/IPython/nbconvert/preprocessors/tests/base.py +++ b/IPython/nbconvert/preprocessors/tests/base.py @@ -36,7 +36,7 @@ class PreprocessorTestsBase(TestsBase): nbformat.new_output(output_type="stream", stream="stdout", output_text="d"), nbformat.new_output(output_type="stream", stream="stderr", output_text="e"), nbformat.new_output(output_type="stream", stream="stderr", output_text="f"), - nbformat.new_output(output_type="png", output_png=b'Zw==')] #g + nbformat.new_output(output_type="png", output_png='Zw==')] #g cells=[nbformat.new_code_cell(input="$ e $", prompt_number=1,outputs=outputs), nbformat.new_text_cell('markdown', source="$ e $")] diff --git a/IPython/nbconvert/preprocessors/tests/test_coalescestreams.py b/IPython/nbconvert/preprocessors/tests/test_coalescestreams.py index ad476d8..fd9c14f 100644 --- a/IPython/nbconvert/preprocessors/tests/test_coalescestreams.py +++ b/IPython/nbconvert/preprocessors/tests/test_coalescestreams.py @@ -14,6 +14,8 @@ Module with tests for the coalescestreams preprocessor # Imports #----------------------------------------------------------------------------- +from IPython.nbformat import current as nbformat + from .base import PreprocessorTestsBase from ..coalescestreams import coalesce_streams @@ -35,4 +37,24 @@ class TestCoalesceStreams(PreprocessorTestsBase): self.assertEqual(outputs[1].output_type, "text") self.assertEqual(outputs[2].text, "cd") self.assertEqual(outputs[3].text, "ef") - + + + def test_coalesce_sequenced_streams(self): + """Can the coalesce streams preprocessor merge a sequence of streams?""" + + outputs = [nbformat.new_output(output_type="stream", stream="stdout", output_text="0"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="1"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="2"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="3"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="4"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="5"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="6"), + nbformat.new_output(output_type="stream", stream="stdout", output_text="7")] + cells=[nbformat.new_code_cell(input="# None", prompt_number=1,outputs=outputs)] + worksheets = [nbformat.new_worksheet(name="worksheet1", cells=cells)] + + nb = nbformat.new_notebook(name="notebook1", worksheets=worksheets) + res = self.build_resources() + nb, res = coalesce_streams(nb, res) + outputs = nb.worksheets[0].cells[0].outputs + self.assertEqual(outputs[0].text, u'01234567') diff --git a/IPython/nbconvert/preprocessors/tests/test_extractoutput.py b/IPython/nbconvert/preprocessors/tests/test_extractoutput.py index 4a13ebd..08bd1f2 100644 --- a/IPython/nbconvert/preprocessors/tests/test_extractoutput.py +++ b/IPython/nbconvert/preprocessors/tests/test_extractoutput.py @@ -46,12 +46,14 @@ class TestExtractOutput(PreprocessorTestsBase): nb, res = preprocessor(nb, res) # Check if text was extracted. - assert 'text_filename' in nb.worksheets[0].cells[0].outputs[1] - text_filename = nb.worksheets[0].cells[0].outputs[1]['text_filename'] + output = nb.worksheets[0].cells[0].outputs[1] + assert 'text_filename' in output + text_filename = output['text_filename'] # Check if png was extracted. - assert 'png_filename' in nb.worksheets[0].cells[0].outputs[6] - png_filename = nb.worksheets[0].cells[0].outputs[6]['png_filename'] + output = nb.worksheets[0].cells[0].outputs[6] + assert 'png_filename' in output + png_filename = output['png_filename'] # Verify text output assert text_filename in res['outputs'] diff --git a/IPython/nbconvert/preprocessors/tests/test_sphinx.py b/IPython/nbconvert/preprocessors/tests/test_highlightmagics.py similarity index 53% rename from IPython/nbconvert/preprocessors/tests/test_sphinx.py rename to IPython/nbconvert/preprocessors/tests/test_highlightmagics.py index 06346cb..ec21374 100644 --- a/IPython/nbconvert/preprocessors/tests/test_sphinx.py +++ b/IPython/nbconvert/preprocessors/tests/test_highlightmagics.py @@ -1,5 +1,5 @@ """ -Module with tests for the sphinx preprocessor +Module with tests for the HighlightMagics preprocessor """ #----------------------------------------------------------------------------- @@ -15,43 +15,54 @@ Module with tests for the sphinx preprocessor #----------------------------------------------------------------------------- from .base import PreprocessorTestsBase -from ..sphinx import SphinxPreprocessor +from ..highlightmagics import HighlightMagicsPreprocessor #----------------------------------------------------------------------------- # Class #----------------------------------------------------------------------------- -class TestSphinx(PreprocessorTestsBase): - """Contains test functions for sphinx.py""" +class TestHighlightMagics(PreprocessorTestsBase): + """Contains test functions for highlightmagics.py""" def build_preprocessor(self): """Make an instance of a preprocessor""" - preprocessor = SphinxPreprocessor() + preprocessor = HighlightMagicsPreprocessor() preprocessor.enabled = True return preprocessor - def test_constructor(self): - """Can a SphinxPreprocessor be constructed?""" + """Can a HighlightMagicsPreprocessor be constructed?""" self.build_preprocessor() - - def test_resources(self): - """Make sure the SphinxPreprocessor adds the appropriate resources to the - resources dict.""" + def test_tagging(self): + """Test the HighlightMagicsPreprocessor tagging""" + nb = self.build_notebook() + res = self.build_resources() + preprocessor = self.build_preprocessor() + nb.worksheets[0].cells[0].input = """%%R -i x,y -o XYcoef + lm.fit <- lm(y~x) + par(mfrow=c(2,2)) + print(summary(lm.fit)) + plot(lm.fit) + XYcoef <- coef(lm.fit)""" + + nb, res = preprocessor(nb, res) + + assert('magics_language' in nb.worksheets[0].cells[0]['metadata']) + + self.assertEqual(nb.worksheets[0].cells[0]['metadata']['magics_language'], 'r') + + def test_no_false_positive(self): + """Test that HighlightMagicsPreprocessor does not tag false positives""" nb = self.build_notebook() res = self.build_resources() preprocessor = self.build_preprocessor() + nb.worksheets[0].cells[0].input = """# this should not be detected + print(\""" + %%R -i x, y + \""")""" nb, res = preprocessor(nb, res) - assert "author" in res['sphinx'] - assert "version" in res['sphinx'] - assert "release" in res['sphinx'] - assert "date" in res['sphinx'] - assert "chapterstyle" in res['sphinx'] - assert "outputstyle" in res['sphinx'] - assert "centeroutput" in res['sphinx'] - assert "header" in res['sphinx'] - assert "texinputs" in res['sphinx'] - assert "pygment_definitions" in res['sphinx'] + + assert('magics_language' not in nb.worksheets[0].cells[0]['metadata']) \ No newline at end of file diff --git a/IPython/nbconvert/templates/html_basic.tpl b/IPython/nbconvert/templates/html_basic.tpl index 0de72d6..1e41c0c 100644 --- a/IPython/nbconvert/templates/html_basic.tpl +++ b/IPython/nbconvert/templates/html_basic.tpl @@ -36,7 +36,7 @@ In [{{ cell.prompt_number }}]: {% block input %}
-{{ cell.input | highlight2html }} +{{ cell.input | highlight2html(metadata=cell.metadata) }}
{%- endblock input %} @@ -135,6 +135,12 @@ unknown type {{ cell.type }} {%- endblock -%} +{%- block data_javascript %} + +{%- endblock -%} + {%- block display_data scoped -%}
{{ super() }} diff --git a/IPython/nbconvert/templates/latex/latex_article.tplx b/IPython/nbconvert/templates/latex/latex_article.tplx index 7916be6..be59552 100644 --- a/IPython/nbconvert/templates/latex/latex_article.tplx +++ b/IPython/nbconvert/templates/latex/latex_article.tplx @@ -1,26 +1,17 @@ -((============================================================================ - NBConvert Sphinx-Latex HowTo Template - Purpose: Allow export of PDF friendly Latex inspired by Sphinx HowTo - document style. Most of the is derived directly from Sphinx source. +% Default to the notebook output style +((* if not cell_style is defined *)) + ((* set cell_style = 'style_ipython.tplx' *)) +((* endif *)) - Inheritance: null>display_priority>sphinx +% Inherit from the specified cell style. +((* extends cell_style *)) - ==========================================================================)) -((*- extends 'sphinx.tplx' -*)) +%=============================================================================== +% Latex Article +%=============================================================================== - -((* set parentdocumentclass = 'article' *)) -((* set documentclass = 'howto' *)) - -((* block h1 -*))part((* endblock h1 -*)) -((* block h2 -*))section((* endblock h2 -*)) -((* block h3 -*))subsection((* endblock h3 -*)) -((* block h4 -*))subsubsection((* endblock h4 -*)) -((* block h5 -*))paragraph((* endblock h5 -*)) -((* block h6 -*))subparagraph((* endblock h6 -*)) - -% Diasble table of contents for howto -((* block toc *)) -((* endblock toc *)) +((* block docclass *)) +\documentclass{article} +((* endblock docclass *)) \ No newline at end of file diff --git a/IPython/nbconvert/templates/latex/latex_base.tplx b/IPython/nbconvert/templates/latex/latex_base.tplx new file mode 100644 index 0000000..337eaef --- /dev/null +++ b/IPython/nbconvert/templates/latex/latex_base.tplx @@ -0,0 +1,197 @@ +((= Latex base template (must inherit) +This template builds upon the abstract template, adding common latex output +functions. Figures, data_text, +This template does not define a docclass, the inheriting class must define this.=)) + +((*- extends 'display_priority.tplx' -*)) + +%=============================================================================== +% Abstract overrides +%=============================================================================== + +((* block header *)) + ((* block docclass *))((* endblock docclass *)) + + ((* block packages *)) + \usepackage{graphicx} % Used to insert images + \usepackage{adjustbox} % Used to constrain images to a maximum size + \usepackage{color} % Allow colors to be defined + \usepackage{enumerate} % Needed for markdown enumerations to work + \usepackage{geometry} % Used to adjust the document margins + \usepackage{amsmath} % Equations + \usepackage{amssymb} % Equations + \usepackage[utf8]{inputenc} % Allow utf-8 characters in the tex document + \usepackage{ucs} % Extended unicode (utf-8) support + \usepackage{fancyvrb} % verbatim replacement that allows latex + \usepackage{grffile} % extends the file name processing of package graphics + % to support a larger range + % The hyperref package gives us a pdf with properly built + % internal navigation ('pdf bookmarks' for the table of contents, + % internal cross-reference links, web links for URLs, etc.) + \usepackage{hyperref} + ((* endblock packages *)) + + ((* block definitions *)) + \definecolor{orange}{cmyk}{0,0.4,0.8,0.2} + \definecolor{darkorange}{rgb}{.71,0.21,0.01} + \definecolor{darkgreen}{rgb}{.12,.54,.11} + \definecolor{myteal}{rgb}{.26, .44, .56} + \definecolor{gray}{gray}{0.45} + \definecolor{lightgray}{gray}{.95} + \definecolor{mediumgray}{gray}{.8} + \definecolor{inputbackground}{rgb}{.95, .95, .85} + \definecolor{outputbackground}{rgb}{.95, .95, .95} + \definecolor{traceback}{rgb}{1, .95, .95} + % new ansi colors + \definecolor{brown}{rgb}{0.54,0.27,0.07} + \definecolor{purple}{rgb}{0.5,0.0,0.5} + \definecolor{darkgray}{gray}{0.25} + \definecolor{lightred}{rgb}{1.0,0.39,0.28} + \definecolor{lightgreen}{rgb}{0.48,0.99,0.0} + \definecolor{lightblue}{rgb}{0.53,0.81,0.92} + \definecolor{lightpurple}{rgb}{0.87,0.63,0.87} + \definecolor{lightcyan}{rgb}{0.5,1.0,0.83} + % Define a nice break command that doesn't care if a line doesn't already + % exist. + \def\br{\hspace*{\fill} \\* } + % Math Jax compatability definitions + \def\gt{>} + \def\lt{<} + % Document parameters + ((* block title *))\title{((( resources.metadata.name | escape_latex )))}((* endblock title *)) + ((* block date *))((* endblock date *)) + ((* block author *))((* endblock author *)) + ((* endblock definitions *)) + + ((* block commands *)) + % Prevent overflowing lines due to hard-to-break entities + \sloppy + % Setup hyperref package + \hypersetup{ + breaklinks=true, % so long urls are correctly broken across lines + colorlinks=true, + urlcolor=blue, + linkcolor=darkorange, + citecolor=darkgreen, + } + % Slightly bigger margins than the latex defaults + ((* block margins *)) + \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in} + ((* endblock margins *)) + ((* endblock commands *)) +((* endblock header *)) + +((* block body *)) + \begin{document} + + ((* block predoc *)) + ((* block maketitle *))\maketitle((* endblock maketitle *)) + ((* block abstract *))((* endblock abstract *)) + ((* endblock predoc *)) + + ((( super() ))) + + % Add a bibliography block to the postdoc + ((* block postdoc *)) + ((* block bibliography *))((* endblock bibliography *)) + ((* endblock postdoc *)) + \end{document} +((* endblock body *)) + +%=============================================================================== +% Support blocks +%=============================================================================== + +% Displaying simple data text +((* block data_text *)) + \begin{verbatim} +((( output.text ))) + \end{verbatim} +((* endblock data_text *)) + +% Display python error text as-is +((* block pyerr *)) + \begin{Verbatim}[commandchars=\\\{\}] +((( super() ))) + \end{Verbatim} +((* endblock pyerr *)) +((* block traceback_line *)) + ((( line | indent | strip_ansi | escape_latex ))) +((* endblock traceback_line *)) + +% Display stream ouput with coloring +((* block stream *)) + \begin{Verbatim}[commandchars=\\\{\}] +((( output.text | escape_latex | ansi2latex ))) + \end{Verbatim} +((* endblock stream *)) + +% Display latex +((* block data_latex -*)) + ((*- if output.latex.startswith('$'): -*)) + ((= Replace $ symbols with more explicit, equation block. =)) + \begin{equation*} + ((( output.latex | strip_dollars ))) + \end{equation*} + ((*- else -*)) + ((( output.latex ))) + ((*- endif *)) +((* endblock data_latex *)) + +% Default mechanism for rendering figures +((*- block data_png -*))((( draw_figure(output.png_filename) )))((*- endblock -*)) +((*- block data_jpg -*))((( draw_figure(output.jpeg_filename) )))((*- endblock -*)) +((*- block data_svg -*))((( draw_figure(output.svg_filename) )))((*- endblock -*)) +((*- block data_pdf -*))((( draw_figure(output.pdf_filename) )))((*- endblock -*)) + +% Draw a figure using the graphicx package. +((* macro draw_figure(filename) -*)) +((* set filename = filename | posix_path *)) +((*- block figure scoped -*)) + \begin{center} + \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{((( filename )))} + \end{center} + { \hspace*{\fill} \\} +((*- endblock figure -*)) +((*- endmacro *)) + +% Draw heading cell. Explicitly map different cell levels. +((* block headingcell scoped *)) + + ((* if cell.level == 1 -*)) + ((* block h1 -*))\section((* endblock h1 -*)) + ((* elif cell.level == 2 -*)) + ((* block h2 -*))\subsection((* endblock h2 -*)) + ((* elif cell.level == 3 -*)) + ((* block h3 -*))\subsubsection((* endblock h3 -*)) + ((* elif cell.level == 4 -*)) + ((* block h4 -*))\paragraph((* endblock h4 -*)) + ((* elif cell.level == 5 -*)) + ((* block h5 -*))\subparagraph((* endblock h5 -*)) + ((* elif cell.level == 6 -*)) + ((* block h6 -*))\\*\textit((* endblock h6 -*)) + ((*- endif -*)) + {((( cell.source | replace('\n', ' ') | citation2latex | markdown2latex )))} + +((* endblock headingcell *)) + +% Redirect pyout to display data priority. +((* block pyout scoped *)) + ((* block data_priority scoped *)) + ((( super() ))) + ((* endblock *)) +((* endblock pyout *)) + +% Render markdown +((* block markdowncell scoped *)) + ((( cell.source | citation2latex | markdown2latex ))) +((* endblock markdowncell *)) + +% Spit out the contents of raw cells unmodified +((* block rawcell scoped *)) + ((( cell.source ))) +((* endblock rawcell *)) + +% Don't display unknown types +((* block unknowncell scoped *)) +((* endblock unknowncell *)) diff --git a/IPython/nbconvert/templates/latex/latex_basic.tplx b/IPython/nbconvert/templates/latex/latex_basic.tplx deleted file mode 100644 index ff3303b..0000000 --- a/IPython/nbconvert/templates/latex/latex_basic.tplx +++ /dev/null @@ -1,270 +0,0 @@ -((*- extends 'display_priority.tplx' -*)) - - -\nonstopmode - -((* block in_prompt *)) -((* endblock in_prompt *)) - -((* block output_prompt *)) -((* endblock output_prompt *)) - -((* block codecell *)) -\begin{codecell} -((( super() ))) -\end{codecell} -((* endblock *)) - -((* block input *)) -\begin{codeinput} -\begin{lstlisting} -((( cell.input ))) -\end{lstlisting} -\end{codeinput} -((* endblock input *)) - -((= Those Two are for error displaying -even if the first one seem to do nothing, -it introduces a new line -=)) - -((* block pyerr *)) -\begin{traceback} -\begin{verbatim} -((( super() ))) -\end{verbatim} -\end{traceback} -((* endblock pyerr *)) - -((* block traceback_line *)) -((( line | indent | strip_ansi ))) -((* endblock traceback_line *)) -((= .... =)) - -((*- block output_group -*)) -\begin{codeoutput} -((( super() ))) -\end{codeoutput} -((* endblock *)) - -((*- block data_png -*)) -\begin{center} -\includegraphics[max size={0.7\textwidth}{0.9\textheight}]{((( output.png_filename | posix_path )))} -\par -\end{center} -((*- endblock -*)) - -((*- block data_jpg -*)) -\begin{center} -\includegraphics[max size={0.7\textwidth}{0.9\textheight}]{((( output.jpeg_filename | posix_path )))} -\par -\end{center} -((*- endblock -*)) - -((*- block data_svg -*)) -\begin{center} -\includegraphics[width=0.7\textwidth]{((( output.svg_filename | posix_path )))} -\par -\end{center} -((*- endblock -*)) - -((*- block data_pdf -*)) -\begin{center} -\includegraphics[width=0.7\textwidth]{((( output.pdf_filename | posix_path )))} -\par -\end{center} -((*- endblock -*)) - -((* block pyout *)) -((* block data_priority scoped *)) -((( super() ))) -((* endblock *)) -((* endblock pyout *)) - -((* block data_text *)) -\begin{verbatim} -((( output.text ))) -\end{verbatim} -((* endblock *)) - -((* block data_latex -*)) -((*- if output.latex.startswith('$'): -*)) \begin{equation*} - ((( output.latex | strip_dollars ))) - \end{equation*} -((*- else -*)) - ((( output.latex ))) -((*- endif *)) -((* endblock *)) - -((* block stream *)) -\begin{Verbatim}[commandchars=\\\{\}] -((( output.text | ansi2latex ))) -\end{Verbatim} -((* endblock stream *)) - -((* block markdowncell scoped *)) -((( cell.source | markdown2latex ))) -((* endblock markdowncell *)) - -((* block headingcell scoped -*)) -((( ('#' * cell.level + cell.source) | replace('\n', ' ') | markdown2latex ))) -((* endblock headingcell *)) - -((* block rawcell scoped *)) -((( cell.source | comment_lines ))) -((* endblock rawcell *)) - -((* block unknowncell scoped *)) -unknown type ((( cell.type ))) -((* endblock unknowncell *)) - -((* block body *)) - -((* block bodyBegin *)) -\begin{document} -((* endblock bodyBegin *)) - -((( super() ))) - -((* block bodyEnd *)) -\end{document} -((* endblock bodyEnd *)) -((* endblock body *)) - -((* block header *)) -%% This file was auto-generated by IPython. -%% Conversion from the original notebook file: -%% -\documentclass[11pt,english]{article} - -%% This is the automatic preamble used by IPython. Note that it does *not* -%% include a documentclass declaration, that is added at runtime to the overall -%% document. - -\usepackage{amsmath} -\usepackage{amssymb} -\usepackage{graphicx} -\usepackage{grffile} -\usepackage{ucs} -\usepackage[utf8x]{inputenc} - -% Scale down larger images -\usepackage[export]{adjustbox} - -%fancy verbatim -\usepackage{fancyvrb} -% needed for markdown enumerations to work -\usepackage{enumerate} - -% Slightly bigger margins than the latex defaults -\usepackage{geometry} -\geometry{verbose,tmargin=3cm,bmargin=3cm,lmargin=2.5cm,rmargin=2.5cm} - -% Define a few colors for use in code, links and cell shading -\usepackage{color} -\definecolor{orange}{cmyk}{0,0.4,0.8,0.2} -\definecolor{darkorange}{rgb}{.71,0.21,0.01} -\definecolor{darkgreen}{rgb}{.12,.54,.11} -\definecolor{myteal}{rgb}{.26, .44, .56} -\definecolor{gray}{gray}{0.45} -\definecolor{lightgray}{gray}{.95} -\definecolor{mediumgray}{gray}{.8} -\definecolor{inputbackground}{rgb}{.95, .95, .85} -\definecolor{outputbackground}{rgb}{.95, .95, .95} -\definecolor{traceback}{rgb}{1, .95, .95} - -% new ansi colors -\definecolor{brown}{rgb}{0.54,0.27,0.07} -\definecolor{purple}{rgb}{0.5,0.0,0.5} -\definecolor{darkgray}{gray}{0.25} -\definecolor{lightred}{rgb}{1.0,0.39,0.28} -\definecolor{lightgreen}{rgb}{0.48,0.99,0.0} -\definecolor{lightblue}{rgb}{0.53,0.81,0.92} -\definecolor{lightpurple}{rgb}{0.87,0.63,0.87} -\definecolor{lightcyan}{rgb}{0.5,1.0,0.83} - -% Framed environments for code cells (inputs, outputs, errors, ...). The -% various uses of \unskip (or not) at the end were fine-tuned by hand, so don't -% randomly change them unless you're sure of the effect it will have. -\usepackage{framed} - -% remove extraneous vertical space in boxes -\setlength\fboxsep{0pt} - -% codecell is the whole input+output set of blocks that a Code cell can -% generate. - -% TODO: unfortunately, it seems that using a framed codecell environment breaks -% the ability of the frames inside of it to be broken across pages. This -% causes at least the problem of having lots of empty space at the bottom of -% pages as new frames are moved to the next page, and if a single frame is too -% long to fit on a page, will completely stop latex from compiling the -% document. So unless we figure out a solution to this, we'll have to instead -% leave the codecell env. as empty. I'm keeping the original codecell -% definition here (a thin vertical bar) for reference, in case we find a -% solution to the page break issue. - -%% \newenvironment{codecell}{% -%% \def\FrameCommand{\color{mediumgray} \vrule width 1pt \hspace{5pt}}% -%% \MakeFramed{\vspace{-0.5em}}} -%% {\unskip\endMakeFramed} - -% For now, make this a no-op... -\newenvironment{codecell}{} - - \newenvironment{codeinput}{% - \def\FrameCommand{\colorbox{inputbackground}}% - \MakeFramed{\advance\hsize-\width \FrameRestore}} - {\unskip\endMakeFramed} - -\newenvironment{codeoutput}{% - \def\FrameCommand{\colorbox{outputbackground}}% - \vspace{-1.4em} - \MakeFramed{\advance\hsize-\width \FrameRestore}} - {\unskip\medskip\endMakeFramed} - -\newenvironment{traceback}{% - \def\FrameCommand{\colorbox{traceback}}% - \MakeFramed{\advance\hsize-\width \FrameRestore}} - {\endMakeFramed} - -% Use and configure listings package for nicely formatted code -\usepackage{listingsutf8} -\lstset{ - language=python, - inputencoding=utf8x, - extendedchars=\true, - aboveskip=\smallskipamount, - belowskip=\smallskipamount, - xleftmargin=2mm, - breaklines=true, - basicstyle=\small \ttfamily, - showstringspaces=false, - keywordstyle=\color{blue}\bfseries, - commentstyle=\color{myteal}, - stringstyle=\color{darkgreen}, - identifierstyle=\color{darkorange}, - columns=fullflexible, % tighter character kerning, like verb -} - -% The hyperref package gives us a pdf with properly built -% internal navigation ('pdf bookmarks' for the table of contents, -% internal cross-reference links, web links for URLs, etc.) -\usepackage{hyperref} -\hypersetup{ - breaklinks=true, % so long urls are correctly broken across lines - colorlinks=true, - urlcolor=blue, - linkcolor=darkorange, - citecolor=darkgreen, - } - -% hardcode size of all verbatim environments to be a bit smaller -\makeatletter -\g@addto@macro\@verbatim\small\topsep=0.5em\partopsep=0pt -\makeatother - -% Prevent overflowing lines due to urls and other hard-to-break entities. -\sloppy - -((* endblock *)) diff --git a/IPython/nbconvert/templates/latex/latex_book.tplx b/IPython/nbconvert/templates/latex/latex_book.tplx deleted file mode 100644 index e9e7ca6..0000000 --- a/IPython/nbconvert/templates/latex/latex_book.tplx +++ /dev/null @@ -1,22 +0,0 @@ -((============================================================================ - NBConvert Sphinx-Latex Manual Template - - Purpose: Allow export of PDF friendly Latex inspired by Sphinx Manual - document style. Most of the is derived directly from Sphinx source. - - Inheritance: null>display_priority>sphinx - - ==========================================================================)) - -((*- extends 'sphinx.tplx' -*)) - - -((* set parentdocumentclass = 'report' *)) -((* set documentclass = 'manual' *)) - -((* block h1 -*))part((* endblock h1 -*)) -((* block h2 -*))chapter((* endblock h2 -*)) -((* block h3 -*))section((* endblock h3 -*)) -((* block h4 -*))subsection((* endblock h4 -*)) -((* block h5 -*))subsubsection((* endblock h5 -*)) -((* block h6 -*))paragraph((* endblock h6 -*)) diff --git a/IPython/nbconvert/templates/latex/latex_report.tplx b/IPython/nbconvert/templates/latex/latex_report.tplx new file mode 100644 index 0000000..d15831d --- /dev/null +++ b/IPython/nbconvert/templates/latex/latex_report.tplx @@ -0,0 +1,22 @@ + +% Default to the notebook output style +((* if not cell_style is defined *)) + ((* set cell_style = 'style_ipython.tplx' *)) +((* endif *)) + +% Inherit from the specified cell style. +((* extends cell_style *)) + + +%=============================================================================== +% Latex Book +%=============================================================================== + +((* block predoc *)) + ((( super() ))) + ((* block tableofcontents *))\tableofcontents((* endblock tableofcontents *)) +((* endblock predoc *)) + +((* block docclass *)) +\documentclass{report} +((* endblock docclass *)) diff --git a/IPython/nbconvert/templates/latex/skeleton/display_priority.tplx b/IPython/nbconvert/templates/latex/skeleton/display_priority.tplx index d4a88ee..055c444 100644 --- a/IPython/nbconvert/templates/latex/skeleton/display_priority.tplx +++ b/IPython/nbconvert/templates/latex/skeleton/display_priority.tplx @@ -1,4 +1,7 @@ -((= autogenerated file do not edit =)) +((= Auto-generated template file, DO NOT edit directly! + To edit this file, please refer to ../../skeleton/README.md =)) + + ((*- extends 'null.tplx' -*)) ((=display data priority=)) @@ -30,10 +33,13 @@ ((*- block data_text -*)) ((*- endblock -*)) ((*- endif -*)) - ((*- if type in ['latex']*)) ((*- block data_latex -*)) ((*- endblock -*)) ((*- endif -*)) + ((*- if type in ['javascript']*)) + ((*- block data_javascript -*)) + ((*- endblock -*)) + ((*- endif -*)) ((*- endfor -*)) ((*- endblock data_priority -*)) diff --git a/IPython/nbconvert/templates/latex/skeleton/null.tplx b/IPython/nbconvert/templates/latex/skeleton/null.tplx index 7c14be9..33bc591 100644 --- a/IPython/nbconvert/templates/latex/skeleton/null.tplx +++ b/IPython/nbconvert/templates/latex/skeleton/null.tplx @@ -1,11 +1,14 @@ -((= autogenerated file do not edit =)) +((= Auto-generated template file, DO NOT edit directly! + To edit this file, please refer to ../../skeleton/README.md =)) + + ((= -DO NOT USE THIS AS A BASE WORK, +DO NOT USE THIS AS A BASE, IF YOU ARE COPY AND PASTING THIS FILE -YOU ARE PROBABLY DOING THINGS WRONG. +YOU ARE PROBABLY DOING THINGS INCORRECTLY. -Null template, Does nothing except defining a basic structure +Null template, does nothing except defining a basic structure To layout the different blocks of a notebook. Subtemplates can override blocks to define their custom representation. diff --git a/IPython/nbconvert/templates/latex/sphinx.tplx b/IPython/nbconvert/templates/latex/sphinx.tplx deleted file mode 100644 index ee59ad8..0000000 --- a/IPython/nbconvert/templates/latex/sphinx.tplx +++ /dev/null @@ -1,469 +0,0 @@ -((= NBConvert Sphinx-Latex Template - -Purpose: Allow export of PDF friendly Latex inspired by Sphinx. Most of the - template is derived directly from Sphinx source. - -Inheritance: null>display_priority - -Note: For best display, use latex syntax highlighting. =)) - -((*- extends 'display_priority.tplx' -*)) - - -\nonstopmode - -%============================================================================== -% Declarations -%============================================================================== - -% In order to make sure that the input/output header follows the code it -% preceeds, the needspace package is used to request that a certain -% amount of lines (specified by this variable) are reserved. If those -% lines aren't available on the current page, the documenter will break -% to the next page and the header along with accomanying lines will be -% rendered together. This value specifies the number of lines that -% the header will be forced to group with without a page break. -((*- set min_header_lines = 4 -*)) - -% This is the number of characters that are permitted per line. It's -% important that this limit is set so characters do not run off the -% edges of latex pages (since latex does not always seem smart enough -% to prevent this in some cases.) This is only applied to textual output -((* if resources.sphinx.outputstyle == 'simple' *)) - ((*- set wrap_size = 85 -*)) -((* elif resources.sphinx.outputstyle == 'notebook' *)) - ((*- set wrap_size = 70 -*)) -((* endif *)) - -%============================================================================== -% Header -%============================================================================== -((* block header *)) - -% Header, overrides base - - % Make sure that the sphinx doc style knows who it inherits from. - \def\sphinxdocclass{(((parentdocumentclass)))} - - % Declare the document class - \documentclass[letterpaper,10pt,english]{((( resources.sphinx.texinputs | posix_path )))/sphinx(((documentclass)))} - - % Imports - \usepackage[utf8]{inputenc} - \DeclareUnicodeCharacter{00A0}{\\nobreakspace} - \usepackage[T1]{fontenc} - \usepackage{babel} - \usepackage{times} - \usepackage{import} - \usepackage[((( resources.sphinx.chapterstyle )))]{((( resources.sphinx.texinputs | posix_path )))/fncychap} - \usepackage{longtable} - \usepackage{((( resources.sphinx.texinputs | posix_path )))/sphinx} - \usepackage{multirow} - - \usepackage{amsmath} - \usepackage{amssymb} - \usepackage{ucs} - \usepackage{enumerate} - - % Used to make the Input/Output rules follow around the contents. - \usepackage{needspace} - - % Pygments requirements - \usepackage{fancyvrb} - \usepackage{color} - % ansi colors additions - \definecolor{darkgreen}{rgb}{.12,.54,.11} - \definecolor{lightgray}{gray}{.95} - \definecolor{brown}{rgb}{0.54,0.27,0.07} - \definecolor{purple}{rgb}{0.5,0.0,0.5} - \definecolor{darkgray}{gray}{0.25} - \definecolor{lightred}{rgb}{1.0,0.39,0.28} - \definecolor{lightgreen}{rgb}{0.48,0.99,0.0} - \definecolor{lightblue}{rgb}{0.53,0.81,0.92} - \definecolor{lightpurple}{rgb}{0.87,0.63,0.87} - \definecolor{lightcyan}{rgb}{0.5,1.0,0.83} - - % Needed to box output/input - \usepackage{tikz} - \usetikzlibrary{calc,arrows,shadows} - \usepackage[framemethod=tikz]{mdframed} - - \usepackage{alltt} - - % Used to load and display graphics - \usepackage{graphicx} - \graphicspath{ {figs/} } - \usepackage[Export]{adjustbox} % To resize - - % used so that images for notebooks which have spaces in the name can still be included - \usepackage{grffile} - - - % For formatting output while also word wrapping. - \usepackage{listings} - \lstset{breaklines=true} - \lstset{basicstyle=\small\ttfamily} - \def\smaller{\fontsize{9.5pt}{9.5pt}\selectfont} - - %Pygments definitions - ((( resources.sphinx.pygment_definitions ))) - - %Set pygments styles if needed... - ((* if resources.sphinx.outputstyle == 'notebook' *)) - \definecolor{nbframe-border}{rgb}{0.867,0.867,0.867} - \definecolor{nbframe-bg}{rgb}{0.969,0.969,0.969} - \definecolor{nbframe-in-prompt}{rgb}{0.0,0.0,0.502} - \definecolor{nbframe-out-prompt}{rgb}{0.545,0.0,0.0} - - \newenvironment{ColorVerbatim} - {\begin{mdframed}[% - roundcorner=1.0pt, % - backgroundcolor=nbframe-bg, % - userdefinedwidth=1\linewidth, % - leftmargin=0.1\linewidth, % - innerleftmargin=0pt, % - innerrightmargin=0pt, % - linecolor=nbframe-border, % - linewidth=1pt, % - usetwoside=false, % - everyline=true, % - innerlinewidth=3pt, % - innerlinecolor=nbframe-bg, % - middlelinewidth=1pt, % - middlelinecolor=nbframe-bg, % - outerlinewidth=0.5pt, % - outerlinecolor=nbframe-border, % - needspace=0pt - ]} - {\end{mdframed}} - - \newenvironment{InvisibleVerbatim} - {\begin{mdframed}[leftmargin=0.1\linewidth,innerleftmargin=3pt,innerrightmargin=3pt, userdefinedwidth=1\linewidth, linewidth=0pt, linecolor=white, usetwoside=false]} - {\end{mdframed}} - - \renewenvironment{Verbatim}[1][\unskip] - {\begin{alltt}\smaller} - {\end{alltt}} - ((* endif *)) - - % Help prevent overflowing lines due to urls and other hard-to-break - % entities. This doesn't catch everything... - \sloppy - - % Document level variables - \title{((( resources.metadata.name | escape_latex )))} - \date{((( resources.sphinx.date | escape_latex )))} - \release{((( resources.sphinx.version | escape_latex )))} - \author{((( resources.sphinx.author | escape_latex )))} - \renewcommand{\releasename}{((( resources.sphinx.release | escape_latex )))} - - % TODO: Add option for the user to specify a logo for his/her export. - \newcommand{\sphinxlogo}{} - - % Make the index page of the document. - \makeindex - - % Import sphinx document type specifics. - ((* block sphinxheader *))((* endblock sphinxheader *)) -((* endblock header *)) - -%============================================================================== -% Body -%============================================================================== -((* block body *)) -((* block bodyBegin *)) -% Body - - % Start of the document - \begin{document} - - ((* if resources.sphinx.header *)) - \maketitle - ((* endif *)) - - ((* block toc *)) - \tableofcontents - ((* endblock toc *)) - - ((* endblock bodyBegin *)) - ((( super() ))) - ((* block bodyEnd *)) - - \renewcommand{\indexname}{Index} - \printindex - - % End of document - \end{document} -((* endblock bodyEnd *)) -((* endblock body *)) - -%============================================================================== -% Footer -%============================================================================== -((* block footer *)) -((* endblock footer *)) - -%============================================================================== -% Headings -% -% Purpose: Format pynb headers as sphinx headers. Depending on the Sphinx -% style that is active, this will change. Thus sphinx styles will -% override the values here. -%============================================================================== -((* block headingcell -*)) - \ - ((*- if cell.level == 1 -*)) - ((* block h1 -*))part((* endblock h1 -*)) - ((*- elif cell.level == 2 -*)) - ((* block h2 -*))chapter((* endblock h2 -*)) - ((*- elif cell.level == 3 -*)) - ((* block h3 -*))section((* endblock h3 -*)) - ((*- elif cell.level == 4 -*)) - ((* block h4 -*))subsection((* endblock h4 -*)) - ((*- elif cell.level == 5 -*)) - ((* block h5 -*))subsubsection((* endblock h5 -*)) - ((*- elif cell.level == 6 -*)) - ((* block h6 -*))paragraph((* endblock h6 -*)) - - ((= It's important to make sure that underscores (which tend to be common - in IPYNB file titles) do not make their way into latex. Sometimes this - causes latex to barf. =)) - ((*- endif -*)) - {((( cell.source | markdown2latex )))} -((*- endblock headingcell *)) - -%============================================================================== -% Markdown -% -% Purpose: Convert markdown to latex. Here markdown2latex is explicitly -% called since we know we want latex output. -%============================================================================== -((*- block markdowncell scoped-*)) -((( cell.source | markdown2latex ))) -((*- endblock markdowncell -*)) - -%============================================================================== -% Rawcell -% -% Purpose: Raw text cells allow the user to manually inject document code that -% will not get touched by the templating system. -%============================================================================== -((*- block rawcell *)) -((( cell.source | wrap_text(wrap_size) ))) -((* endblock rawcell -*)) - -%============================================================================== -% Unknowncell -% -% Purpose: This is the catch anything unhandled. To display this data, we -% remove all possible latex conflicts and wrap the characters so they -% can't flow off of the page. -%============================================================================== -((* block unknowncell scoped*)) -% Unsupported cell type, no formatting -((( cell.source | wrap_text | escape_latex ))) -((* endblock unknowncell *)) - -%============================================================================== -% Input -%============================================================================== -((* block input *)) - - % Make sure that atleast 4 lines are below the HR - \needspace{((( min_header_lines )))\baselineskip} - - ((* if resources.sphinx.outputstyle == 'simple' *)) - - % Add a horizantal break, along with break title. - \vspace{10pt} - {\scriptsize Input}\\* - \rule[10pt]{\linewidth}{0.5pt} - \vspace{-25pt} - - % Add contents below. - ((( cell.input | highlight2latex ))) - - ((* elif resources.sphinx.outputstyle == 'notebook' *)) - \vspace{6pt} - ((( write_prompt("In", cell.prompt_number, "nbframe-in-prompt") ))) - \vspace{-2.65\baselineskip} - \begin{ColorVerbatim} - \vspace{-0.7\baselineskip} - ((( cell.input | highlight2latex ))) - ((* if cell.input == None or cell.input == '' *)) - \vspace{0.3\baselineskip} - ((* else *)) - \vspace{-0.2\baselineskip} - ((* endif *)) - \end{ColorVerbatim} - ((* endif *)) -((* endblock input *)) - -%============================================================================== -% Output_Group -% -% Purpose: Make sure that only one header bar only attaches to the output -% once. By keeping track of when an input group is started -%============================================================================== -((* block output_group *)) - ((* if cell.outputs.__len__() > 0 *)) - - % If the first block is an image, minipage the image. Else - % request a certain amount of space for the input text. - ((( iff_figure(cell.outputs[0], "\\begin{minipage}{1.0\\textwidth}", "\\needspace{" ~ min_header_lines ~ "\\baselineskip}") ))) - - ((* if resources.sphinx.outputstyle == 'simple' *)) - - % Add a horizantal break, along with break title. - \vspace{10pt} - {\scriptsize Output}\\* - \rule[10pt]{\linewidth}{0.5pt} - \vspace{-20pt} - - % Add the contents of the first block. - ((( render_output(cell.outputs[0]) ))) - - % Close the minipage. - ((( iff_figure(cell.outputs[0], "\\end{minipage}", "") ))) - - % Add remainer of the document contents below. - ((* for output in cell.outputs[1:] *)) - ((( render_output(output, cell.prompt_number) ))) - ((* endfor *)) - ((* elif resources.sphinx.outputstyle == 'notebook' *)) - - % Add document contents. - ((* for output in cell.outputs *)) - ((( render_output(output, cell.prompt_number) ))) - ((* endfor *)) - ((* endif *)) - ((* endif *)) -((* endblock *)) - -%============================================================================== -% Additional formating -%============================================================================== -((* block data_text *)) -((( custom_verbatim(output.text) | ansi2latex ))) -((* endblock *)) - -((* block traceback_line *)) -((( conditionally_center_output( line | indent| strip_ansi ) ))) -((* endblock traceback_line *)) - -%============================================================================== -% Supported image formats -%============================================================================== -((*- block data_png -*)) -((( conditionally_center_output(insert_graphics(output.png_filename | posix_path)) ))) -((*- endblock -*)) - -((*- block data_jpg -*)) -((( conditionally_center_output(insert_graphics(output.jpg_filename | posix_path)) ))) -((*- endblock -*)) - -((*- block data_svg -*)) -((( conditionally_center_output(insert_graphics(output.svg_filename | posix_path)) ))) -((*- endblock -*)) - -((*- block data_pdf -*)) -((( conditionally_center_output(insert_graphics(output.pdf_filename | posix_path)) ))) -((*- endblock -*)) - -((*- block data_latex *)) -((* if resources.sphinx.centeroutput *)) - \begin{center} -((* endif -*)) -((( output.latex | strip_math_space ))) -((*- if resources.sphinx.centeroutput *)) - \end{center} -((* endif -*)) -((*- endblock -*)) - -%============================================================================== -% Support Macros -%============================================================================== - -% Name: write_prompt -% Purpose: Renders an output/input prompt for notebook style pdfs -((* macro write_prompt(prompt, number, color) -*)) - \makebox[0.1\linewidth]{\smaller\hfill\tt\color{((( color )))}((( prompt )))\hspace{4pt}{[}((( number ))){]}:\hspace{4pt}}\\* -((*- endmacro *)) - -% Name: render_output -% Purpose: Renders an output block appropriately. -((* macro render_output(output, prompt_number) -*)) - ((*- if output.output_type == 'pyerr' -*)) - ((*- block pyerr scoped *)) - ((( custom_verbatim(super()) ))) - ((* endblock pyerr -*)) - ((*- else -*)) - - ((* if resources.sphinx.outputstyle == 'notebook' *)) - ((*- if output.output_type == 'pyout' -*)) - ((( write_prompt("Out", prompt_number, "nbframe-out-prompt") ))) - \vspace{-2.55\baselineskip} - ((*- endif -*)) - - \begin{InvisibleVerbatim} - \vspace{-0.5\baselineskip} - ((*- endif -*)) - - ((*- block display_data scoped -*)) - ((( super() ))) - ((*- endblock display_data -*)) - - ((* if resources.sphinx.outputstyle == 'notebook' *)) - \end{InvisibleVerbatim} - ((*- endif -*)) - ((*- endif -*)) -((*- endmacro *)) - -% Name: iff_figure -% Purpose: If the output block provided is a figure type, the 'true_content' -% parameter will be returned. Else, the 'false_content'. -((* macro iff_figure(output, true_content, false_content) -*)) - ((*- set is_figure = false -*)) - ((*- for type in output | filter_data_type -*)) - ((*- if type in ['pdf', 'svg', 'png', 'jpeg','html']*)) - ((*- set is_figure = true -*)) - ((*- endif -*)) - ((*- endfor -*)) - - ((* if is_figure -*)) - ((( true_content ))) - ((*- else -*)) - ((( false_content ))) - ((*- endif *)) -((*- endmacro *)) - -% Name: custom_verbatim -% Purpose: This macro creates a verbatim style block that fits the existing -% sphinx style more readily than standard verbatim blocks. -((* macro custom_verbatim(text) -*)) - \begin{alltt} - ((*- if resources.sphinx.centeroutput *))\begin{center} ((* endif -*)) -((( text | wrap_text(wrap_size) | escape_latex ))) - ((*- if resources.sphinx.centeroutput *))\end{center}((* endif -*)) - \end{alltt} -((*- endmacro *)) - -% Name: conditionally_center_output -% Purpose: This macro centers the output if the output centering is enabled. -((* macro conditionally_center_output(text) -*)) - ((* if resources.sphinx.centeroutput *)) - {\centering - ((* endif *)) - ((( text ))) - ((* if resources.sphinx.centeroutput *))} - ((* endif *)) -((*- endmacro *)) - -% Name: insert_graphics -% Purpose: This macro will insert an image in the latex document given a path. -((* macro insert_graphics(path) -*)) - \begin{center} - \includegraphics[max size={\textwidth}{\textheight}]{((( path )))} - \par - \end{center} -((*- endmacro *)) diff --git a/IPython/nbconvert/templates/latex/style_bw_ipython.tplx b/IPython/nbconvert/templates/latex/style_bw_ipython.tplx new file mode 100644 index 0000000..3df4556 --- /dev/null +++ b/IPython/nbconvert/templates/latex/style_bw_ipython.tplx @@ -0,0 +1,41 @@ +((= Black&white ipython input/output style =)) + +((*- extends 'latex_base.tplx' -*)) + +%=============================================================================== +% Input +%=============================================================================== + +((* block input scoped *)) +((( add_prompt(cell.input, cell, 'In ') ))) +((* endblock input *)) + + +%=============================================================================== +% Output +%=============================================================================== + +((* block pyout scoped *)) + ((*- for type in output | filter_data_type -*)) + ((*- if type in ['text']*)) +((( add_prompt(output.text, cell, 'Out') ))) + ((*- else -*)) +\verb+Out[((( cell.prompt_number )))]:+((( super() ))) + ((*- endif -*)) + ((*- endfor -*)) +((* endblock pyout *)) + + +%============================================================================== +% Support Macros +%============================================================================== + +% Name: draw_prompt +% Purpose: Renders an output/input prompt +((* macro add_prompt(text, cell, prompt) -*)) + ((*- set prompt_number = "" ~ cell.prompt_number -*)) + ((*- set indentation = " " * (prompt_number | length + 7) -*)) +\begin{verbatim} +(((- text | add_prompts(first=prompt ~ '[' ~ prompt_number ~ ']: ', cont=indentation) -))) +\end{verbatim} +((*- endmacro *)) diff --git a/IPython/nbconvert/templates/latex/style_bw_python.tplx b/IPython/nbconvert/templates/latex/style_bw_python.tplx new file mode 100644 index 0000000..5371025 --- /dev/null +++ b/IPython/nbconvert/templates/latex/style_bw_python.tplx @@ -0,0 +1,13 @@ +((= Black&white Python input/output style =)) + +((*- extends 'latex_base.tplx' -*)) + +%=============================================================================== +% Input +%=============================================================================== + +((* block input scoped *)) +\begin{verbatim} +((( cell.input | add_prompts ))) +\end{verbatim} +((* endblock input *)) diff --git a/IPython/nbconvert/templates/latex/style_ipython.tplx b/IPython/nbconvert/templates/latex/style_ipython.tplx new file mode 100644 index 0000000..77bc0b9 --- /dev/null +++ b/IPython/nbconvert/templates/latex/style_ipython.tplx @@ -0,0 +1,54 @@ +((= IPython input/output style =)) + +((*- extends 'latex_base.tplx' -*)) + +% Custom definitions +((* block definitions *)) + ((( super() ))) + + % Pygments definitions + ((( resources.latex.pygments_definitions ))) + + % Exact colors from NB + \definecolor{incolor}{rgb}{0.0, 0.0, 0.5} + \definecolor{outcolor}{rgb}{0.545, 0.0, 0.0} + +((* endblock definitions *)) + +%=============================================================================== +% Input +%=============================================================================== + +((* block input scoped *)) + ((( add_prompt(cell.input | highlight2latex(strip_verbatim=True), cell, 'In ', 'incolor') ))) +((* endblock input *)) + + +%=============================================================================== +% Output +%=============================================================================== + +((* block pyout scoped *)) + ((*- for type in output | filter_data_type -*)) + ((*- if type in ['text']*)) + ((( add_prompt(output.text | escape_latex, cell, 'Out', 'outcolor') ))) + ((* else -*)) +\texttt{\color{outcolor}Out[{\color{outcolor}((( cell.prompt_number )))}]:}((( super() ))) + ((*- endif -*)) + ((*- endfor -*)) +((* endblock pyout *)) + + +%============================================================================== +% Support Macros +%============================================================================== + +% Name: draw_prompt +% Purpose: Renders an output/input prompt +((* macro add_prompt(text, cell, prompt, prompt_color) -*)) + ((*- set prompt_number = "" ~ cell.prompt_number -*)) + ((*- set indention = " " * (prompt_number | length + 7) -*)) +\begin{Verbatim}[commandchars=\\\{\}] +((( text | add_prompts(first='{\color{' ~ prompt_color ~ '}' ~ prompt ~ '[{\\color{' ~ prompt_color ~ '}' ~ prompt_number ~ '}]:} ', cont=indention) ))) +\end{Verbatim} +((*- endmacro *)) diff --git a/IPython/nbconvert/templates/latex/style_python.tplx b/IPython/nbconvert/templates/latex/style_python.tplx new file mode 100644 index 0000000..f6f6e87 --- /dev/null +++ b/IPython/nbconvert/templates/latex/style_python.tplx @@ -0,0 +1,21 @@ +((= Python input/output style =)) + +((*- extends 'latex_base.tplx' -*)) + +% Custom definitions +((* block definitions *)) + ((( super() ))) + + % Pygments definitions + ((( resources.latex.pygments_definitions ))) +((* endblock definitions *)) + +%=============================================================================== +% Input +%=============================================================================== + +((* block input scoped *)) + \begin{Verbatim}[commandchars=\\\{\}] +((( cell.input | highlight2latex(strip_verbatim=True) | add_prompts ))) + \end{Verbatim} +((* endblock input *)) diff --git a/IPython/nbconvert/templates/markdown.tpl b/IPython/nbconvert/templates/markdown.tpl index a759470..399c012 100644 --- a/IPython/nbconvert/templates/markdown.tpl +++ b/IPython/nbconvert/templates/markdown.tpl @@ -2,19 +2,13 @@ {% block in_prompt %} -In[{{ cell.prompt_number if cell.prompt_number else ' ' }}]: {% endblock in_prompt %} {% block output_prompt %} -{% if cell.haspyout %} -Out[{{ cell.prompt_number }}]: -{%- endif %} {%- endblock output_prompt %} {% block input %} -``` -{{ cell.input }} -``` +{{ cell.input | indent(4)}} {% endblock input %} {% block pyerr %} @@ -26,6 +20,7 @@ Out[{{ cell.prompt_number }}]: {% endblock traceback_line %} {% block pyout %} + {% block data_priority scoped %} {{ super() }} {% endblock %} @@ -36,23 +31,25 @@ Out[{{ cell.prompt_number }}]: {% endblock stream %} {% block data_svg %} -[!image]({{ output.svg_filename }}) +![svg]({{ output.svg_filename | path2url }}) {% endblock data_svg %} {% block data_png %} -[!image]({{ output.png_filename }}) +![png]({{ output.png_filename | path2url }}) {% endblock data_png %} {% block data_jpg %} -[!image]({{ output.jpg_filename }}) +![jpeg]({{ output.jpeg_filename | path2url }}) {% endblock data_jpg %} {% block data_latex %} -$$ {{ output.latex }} -$$ {% endblock data_latex %} +{% block data_html scoped %} +{{ output.html }} +{% endblock data_html %} + {% block data_text scoped %} {{ output.text | indent }} {% endblock data_text %} @@ -61,6 +58,7 @@ $$ {{ cell.source | wrap_text(80) }} {% endblock markdowncell %} + {% block headingcell scoped %} {{ '#' * cell.level }} {{ cell.source | replace('\n', ' ') }} {% endblock headingcell %} diff --git a/IPython/nbconvert/templates/rst.tpl b/IPython/nbconvert/templates/rst.tpl index 463a0e8..d0a7fc2 100644 --- a/IPython/nbconvert/templates/rst.tpl +++ b/IPython/nbconvert/templates/rst.tpl @@ -2,25 +2,22 @@ {% block in_prompt %} - -In[{{ cell.prompt_number if cell.prompt_number else ' ' }}]: - -.. code:: python - {% endblock in_prompt %} {% block output_prompt %} -{% if cell.haspyout -%} - Out[{{ cell.prompt_number }}]: -{% endif %} {% endblock output_prompt %} {% block input %} +{%- if not cell.input.isspace() -%} +.. code:: python + {{ cell.input | indent}} +{%- endif -%} {% endblock input %} {% block pyerr %} :: + {{ super() }} {% endblock pyerr %} @@ -49,19 +46,27 @@ In[{{ cell.prompt_number if cell.prompt_number else ' ' }}]: {% endblock data_png %} {% block data_jpg %} -..jpg image:: {{ output.jpg_filename }} +.. image:: {{ output.jpeg_filename }} {% endblock data_jpg %} {% block data_latex %} .. math:: -{{ output.latex | indent }} + +{{ output.latex | strip_dollars | indent }} {% endblock data_latex %} {% block data_text scoped %} .. parsed-literal:: + {{ output.text | indent }} {% endblock data_text %} +{% block data_html scoped %} +.. raw:: html + +{{ output.html | indent }} +{% endblock data_html %} + {% block markdowncell scoped %} {{ cell.source | markdown2rst }} {% endblock markdowncell %} diff --git a/IPython/nbconvert/templates/skeleton/Makefile b/IPython/nbconvert/templates/skeleton/Makefile index bb9b933..c02ac0f 100644 --- a/IPython/nbconvert/templates/skeleton/Makefile +++ b/IPython/nbconvert/templates/skeleton/Makefile @@ -1,12 +1,14 @@ +TPLS := $(patsubst %.tpl,../latex/skeleton/%.tplx,$(wildcard *.tpl)) +all: clean $(TPLS) -all: tex/null.tplx tex/display_priority.tplx - -# convert jinja syntax to tex -# cf http://flask.pocoo.org/snippets/55/ -tex/%.tplx: %.tpl +# Convert standard Jinja2 syntax to LaTeX safe Jinja2 +# see http://flask.pocoo.org/snippets/55/ for more info +../latex/skeleton/%.tplx: %.tpl @echo 'generating tex equivalent of $^: $@' - @echo '((= autogenerated file do not edit =))' > $@ + @echo '((= Auto-generated template file, DO NOT edit directly!\n' \ + ' To edit this file, please refer to ../../skeleton/README.md' \ + '=))\n\n' > $@ @sed \ -e 's/{%/((*/g' \ -e 's/%}/*))/g' \ @@ -17,7 +19,6 @@ tex/%.tplx: %.tpl -e "s/tpl'/tplx'/g" \ $^ >> $@ - clean: @echo "cleaning generated tplx files..." - @rm tex/* + @-rm ../latex/skeleton/*.tplx diff --git a/IPython/nbconvert/templates/skeleton/README.md b/IPython/nbconvert/templates/skeleton/README.md index cda024f..22d4499 100644 --- a/IPython/nbconvert/templates/skeleton/README.md +++ b/IPython/nbconvert/templates/skeleton/README.md @@ -1,6 +1,8 @@ ## Template skeleton -This contain skeleton template that you probably don't want -to inherit directly. +This directory contains the template skeleton files. -do not modify the content of the 'tex' folder which is generated by running 'make' in this folder. +Do not modify the contents of the `../latex/skeleton` folder. Instead, +if you need to, make modifications to the files in this folder and then run +`make` to generate the corresponding latex skeleton files in the +`../latex/skeleton` folder. \ No newline at end of file diff --git a/IPython/nbconvert/templates/skeleton/display_priority.tpl b/IPython/nbconvert/templates/skeleton/display_priority.tpl index f237e55..5299d48 100644 --- a/IPython/nbconvert/templates/skeleton/display_priority.tpl +++ b/IPython/nbconvert/templates/skeleton/display_priority.tpl @@ -29,10 +29,13 @@ {%- block data_text -%} {%- endblock -%} {%- endif -%} - {%- if type in ['latex']%} {%- block data_latex -%} {%- endblock -%} {%- endif -%} + {%- if type in ['javascript']%} + {%- block data_javascript -%} + {%- endblock -%} + {%- endif -%} {%- endfor -%} {%- endblock data_priority -%} diff --git a/IPython/nbconvert/templates/skeleton/null.tpl b/IPython/nbconvert/templates/skeleton/null.tpl index b0d4f3d..24066f2 100644 --- a/IPython/nbconvert/templates/skeleton/null.tpl +++ b/IPython/nbconvert/templates/skeleton/null.tpl @@ -1,10 +1,10 @@ {# -DO NOT USE THIS AS A BASE WORK, +DO NOT USE THIS AS A BASE, IF YOU ARE COPY AND PASTING THIS FILE -YOU ARE PROBABLY DOING THINGS WRONG. +YOU ARE PROBABLY DOING THINGS INCORRECTLY. -Null template, Does nothing except defining a basic structure +Null template, does nothing except defining a basic structure To layout the different blocks of a notebook. Subtemplates can override blocks to define their custom representation. diff --git a/IPython/nbconvert/templates/slides_reveal.tpl b/IPython/nbconvert/templates/slides_reveal.tpl index 72444c5..2581c4d 100644 --- a/IPython/nbconvert/templates/slides_reveal.tpl +++ b/IPython/nbconvert/templates/slides_reveal.tpl @@ -135,7 +135,7 @@ transition: Reveal.getQueryHash().transition || 'linear', // default/cube/page/c dependencies: [ { src: "{{resources.reveal.url_prefix}}/lib/js/classList.js", condition: function() { return !document.body.classList; } }, { src: "{{resources.reveal.url_prefix}}/plugin/highlight/highlight.js", async: true, callback: function() { hljs.initHighlightingOnLoad(); } }, -{ src: "{{resources.reveal.notes_prefix}}/plugin/notes/notes.js", async: true, condition: function() { return !!document.body.classList; } } +{ src: "{{resources.reveal.url_prefix}}/plugin/notes/notes.js", async: true, condition: function() { return !!document.body.classList; } } // { src: 'http://s7.addthis.com/js/300/addthis_widget.js', async: true}, ] }); diff --git a/IPython/nbconvert/tests/base.py b/IPython/nbconvert/tests/base.py index 81df37b..c84cde8 100644 --- a/IPython/nbconvert/tests/base.py +++ b/IPython/nbconvert/tests/base.py @@ -16,12 +16,12 @@ Contains base test class for nbconvert import os import glob import shutil +import unittest import IPython from IPython.utils.tempdir import TemporaryWorkingDirectory from IPython.utils.process import get_output_error_code from IPython.testing.tools import get_ipython_cmd -from IPython.testing.ipunittest import ParametricTestCase # a trailing space allows for simpler concatenation with the other arguments ipy_cmd = get_ipython_cmd(as_string=True) + " " @@ -31,7 +31,7 @@ ipy_cmd = get_ipython_cmd(as_string=True) + " " #----------------------------------------------------------------------------- -class TestsBase(ParametricTestCase): +class TestsBase(unittest.TestCase): """Base tests class. Contains useful fuzzy comparison and nbconvert functions.""" @@ -86,7 +86,7 @@ class TestsBase(ParametricTestCase): For example: Replace "ii" with "i" in the string "Hiiii" yields "Hii" - Another replacement yields "Hi" (the desired output) + Another replacement cds "Hi" (the desired output) Parameters: ----------- diff --git a/IPython/nbconvert/tests/files/notebook2.ipynb b/IPython/nbconvert/tests/files/notebook2.ipynb index a7fe1a5..807a2a2 100644 --- a/IPython/nbconvert/tests/files/notebook2.ipynb +++ b/IPython/nbconvert/tests/files/notebook2.ipynb @@ -124,6 +124,14 @@ "prompt_number": 10 }, { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Here is a very long heading that pandoc will wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap and wrap" + ] + }, + { "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/IPython/nbconvert/tests/test_nbconvertapp.py b/IPython/nbconvert/tests/test_nbconvertapp.py index 1eabe13..4df503e 100644 --- a/IPython/nbconvert/tests/test_nbconvertapp.py +++ b/IPython/nbconvert/tests/test_nbconvertapp.py @@ -1,12 +1,10 @@ -""" -Contains tests for the nbconvertapp -""" +"""Test NbConvertApp""" + #----------------------------------------------------------------------------- -#Copyright (c) 2013, the IPython Development Team. -# -#Distributed under the terms of the Modified BSD License. +# Copyright (C) 2013 The IPython Development Team # -#The full license is in the file COPYING.txt, distributed with this software. +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- @@ -15,12 +13,14 @@ Contains tests for the nbconvertapp import os import glob +import sys from .base import TestsBase +import IPython.testing.tools as tt from IPython.testing import decorators as dec - + #----------------------------------------------------------------------------- # Constants #----------------------------------------------------------------------------- @@ -35,13 +35,14 @@ class TestNbConvertApp(TestsBase): def test_notebook_help(self): - """ - Will help show if no notebooks are specified? - """ + """Will help show if no notebooks are specified?""" with self.create_temp_cwd(): out, err = self.call('nbconvert --log-level 0', ignore_return_code=True) - assert "see '--help-all'" in out - + self.assertIn("see '--help-all'", out) + + def test_help_output(self): + """ipython nbconvert --help-all works""" + tt.help_all_output_test('nbconvert') def test_glob(self): """ @@ -102,6 +103,30 @@ class TestNbConvertApp(TestsBase): assert os.path.isfile('notebook1.tex') assert os.path.isfile('notebook1.pdf') + @dec.onlyif_cmds_exist('pandoc') + def test_spurious_cr(self): + """Check for extra CR characters""" + with self.create_temp_cwd(['notebook2.ipynb']): + self.call('nbconvert --log-level 0 --to latex notebook2') + assert os.path.isfile('notebook2.tex') + with open('notebook2.tex') as f: + tex = f.read() + self.call('nbconvert --log-level 0 --to html notebook2') + assert os.path.isfile('notebook2.html') + with open('notebook2.html') as f: + html = f.read() + self.assertEqual(tex.count('\r'), tex.count('\r\n')) + self.assertEqual(html.count('\r'), html.count('\r\n')) + + @dec.onlyif_cmds_exist('pandoc') + def test_png_base64_html_ok(self): + """Is embedded png data well formed in HTML?""" + with self.create_temp_cwd(['notebook2.ipynb']): + self.call('nbconvert --log-level 0 --to HTML ' + 'notebook2.ipynb --template full') + assert os.path.isfile('notebook2.html') + with open('notebook2.html') as f: + assert "data:image/png;base64,b'" not in f.read() @dec.onlyif_cmds_exist('pandoc') def test_template(self): diff --git a/IPython/nbconvert/utils/pandoc.py b/IPython/nbconvert/utils/pandoc.py index 4be14bb..248309a 100644 --- a/IPython/nbconvert/utils/pandoc.py +++ b/IPython/nbconvert/utils/pandoc.py @@ -15,6 +15,7 @@ from __future__ import print_function # Stdlib imports import subprocess +from io import TextIOWrapper, BytesIO # IPython imports from IPython.utils.py3compat import cast_bytes @@ -64,6 +65,6 @@ def pandoc(source, fmt, to, extra_args=None, encoding='utf-8'): "http://johnmacfarlane.net/pandoc/installing.html" ) out, _ = p.communicate(cast_bytes(source, encoding)) - out = out.decode(encoding, 'replace') - return out[:-1] + out = TextIOWrapper(BytesIO(out), encoding, 'replace').read() + return out.rstrip('\n') diff --git a/IPython/nbformat/v3/nbbase.py b/IPython/nbformat/v3/nbbase.py index bac60df..a1b697f 100644 --- a/IPython/nbformat/v3/nbbase.py +++ b/IPython/nbformat/v3/nbbase.py @@ -25,6 +25,7 @@ import pprint import uuid from IPython.utils.ipstruct import Struct +from IPython.utils.py3compat import cast_unicode #----------------------------------------------------------------------------- # Code @@ -67,21 +68,21 @@ def new_output(output_type=None, output_text=None, output_png=None, if output_type != 'pyerr': if output_text is not None: - output.text = unicode(output_text) + output.text = cast_unicode(output_text) if output_png is not None: - output.png = bytes(output_png) + output.png = cast_unicode(output_png) if output_jpeg is not None: - output.jpeg = bytes(output_jpeg) + output.jpeg = cast_unicode(output_jpeg) if output_html is not None: - output.html = unicode(output_html) + output.html = cast_unicode(output_html) if output_svg is not None: - output.svg = unicode(output_svg) + output.svg = cast_unicode(output_svg) if output_latex is not None: - output.latex = unicode(output_latex) + output.latex = cast_unicode(output_latex) if output_json is not None: - output.json = unicode(output_json) + output.json = cast_unicode(output_json) if output_javascript is not None: - output.javascript = unicode(output_javascript) + output.javascript = cast_unicode(output_javascript) if output_type == u'pyout': if prompt_number is not None: @@ -89,14 +90,14 @@ def new_output(output_type=None, output_text=None, output_png=None, if output_type == u'pyerr': if ename is not None: - output.ename = unicode(ename) + output.ename = cast_unicode(ename) if evalue is not None: - output.evalue = unicode(evalue) + output.evalue = cast_unicode(evalue) if traceback is not None: - output.traceback = [unicode(frame) for frame in list(traceback)] + output.traceback = [cast_unicode(frame) for frame in list(traceback)] if output_type == u'stream': - output.stream = 'stdout' if stream is None else unicode(stream) + output.stream = 'stdout' if stream is None else cast_unicode(stream) return output @@ -107,9 +108,9 @@ def new_code_cell(input=None, prompt_number=None, outputs=None, cell = NotebookNode() cell.cell_type = u'code' if language is not None: - cell.language = unicode(language) + cell.language = cast_unicode(language) if input is not None: - cell.input = unicode(input) + cell.input = cast_unicode(input) if prompt_number is not None: cell.prompt_number = int(prompt_number) if outputs is None: @@ -130,9 +131,9 @@ def new_text_cell(cell_type, source=None, rendered=None, metadata=None): if cell_type == 'plaintext': cell_type = 'raw' if source is not None: - cell.source = unicode(source) + cell.source = cast_unicode(source) if rendered is not None: - cell.rendered = unicode(rendered) + cell.rendered = cast_unicode(rendered) cell.metadata = NotebookNode(metadata or {}) cell.cell_type = cell_type return cell @@ -143,9 +144,9 @@ def new_heading_cell(source=None, rendered=None, level=1, metadata=None): cell = NotebookNode() cell.cell_type = u'heading' if source is not None: - cell.source = unicode(source) + cell.source = cast_unicode(source) if rendered is not None: - cell.rendered = unicode(rendered) + cell.rendered = cast_unicode(rendered) cell.level = int(level) cell.metadata = NotebookNode(metadata or {}) return cell @@ -155,7 +156,7 @@ def new_worksheet(name=None, cells=None, metadata=None): """Create a worksheet by name with with a list of cells.""" ws = NotebookNode() if name is not None: - ws.name = unicode(name) + ws.name = cast_unicode(name) if cells is None: ws.cells = [] else: @@ -178,7 +179,7 @@ def new_notebook(name=None, metadata=None, worksheets=None): else: nb.metadata = NotebookNode(metadata) if name is not None: - nb.metadata.name = unicode(name) + nb.metadata.name = cast_unicode(name) return nb @@ -187,29 +188,29 @@ def new_metadata(name=None, authors=None, license=None, created=None, """Create a new metadata node.""" metadata = NotebookNode() if name is not None: - metadata.name = unicode(name) + metadata.name = cast_unicode(name) if authors is not None: metadata.authors = list(authors) if created is not None: - metadata.created = unicode(created) + metadata.created = cast_unicode(created) if modified is not None: - metadata.modified = unicode(modified) + metadata.modified = cast_unicode(modified) if license is not None: - metadata.license = unicode(license) + metadata.license = cast_unicode(license) if gistid is not None: - metadata.gistid = unicode(gistid) + metadata.gistid = cast_unicode(gistid) return metadata def new_author(name=None, email=None, affiliation=None, url=None): """Create a new author.""" author = NotebookNode() if name is not None: - author.name = unicode(name) + author.name = cast_unicode(name) if email is not None: - author.email = unicode(email) + author.email = cast_unicode(email) if affiliation is not None: - author.affiliation = unicode(affiliation) + author.affiliation = cast_unicode(affiliation) if url is not None: - author.url = unicode(url) + author.url = cast_unicode(url) return author diff --git a/IPython/nbformat/v3/nbjson.py b/IPython/nbformat/v3/nbjson.py index 5c63a4e..be9ee28 100644 --- a/IPython/nbformat/v3/nbjson.py +++ b/IPython/nbformat/v3/nbjson.py @@ -46,7 +46,7 @@ class JSONReader(NotebookReader): return nb def to_notebook(self, d, **kwargs): - return restore_bytes(rejoin_lines(from_dict(d))) + return rejoin_lines(from_dict(d)) class JSONWriter(NotebookWriter): diff --git a/IPython/nbformat/v3/nbpy.py b/IPython/nbformat/v3/nbpy.py index 23fc2f0..bd4a9ad 100644 --- a/IPython/nbformat/v3/nbpy.py +++ b/IPython/nbformat/v3/nbpy.py @@ -190,7 +190,7 @@ class PyWriter(NotebookWriter): lines.extend([u'# ' + line for line in input.splitlines()]) lines.append(u'') lines.append('') - return unicode('\n'.join(lines)) + return u'\n'.join(lines) _reader = PyReader() diff --git a/IPython/nbformat/v3/rwbase.py b/IPython/nbformat/v3/rwbase.py index d8c1f17..a381242 100644 --- a/IPython/nbformat/v3/rwbase.py +++ b/IPython/nbformat/v3/rwbase.py @@ -32,6 +32,8 @@ def restore_bytes(nb): Base64 encoding is handled elsewhere. Bytes objects in the notebook are always b64-encoded. We DO NOT encode/decode around file formats. + + Note: this is never used """ for ws in nb.worksheets: for cell in ws.cells: diff --git a/IPython/nbformat/v3/tests/nbexamples.py b/IPython/nbformat/v3/tests/nbexamples.py index 96685ca..713957d 100644 --- a/IPython/nbformat/v3/tests/nbexamples.py +++ b/IPython/nbformat/v3/tests/nbexamples.py @@ -9,9 +9,9 @@ from ..nbbase import ( new_metadata, new_author, new_heading_cell, nbformat, nbformat_minor ) -# some random base64-encoded *bytes* -png = encodestring(os.urandom(5)) -jpeg = encodestring(os.urandom(6)) +# some random base64-encoded *text* +png = encodestring(os.urandom(5)).decode('ascii') +jpeg = encodestring(os.urandom(6)).decode('ascii') ws = new_worksheet(name='worksheet1') diff --git a/IPython/nbformat/v3/tests/test_json.py b/IPython/nbformat/v3/tests/test_json.py index 9897443..fba1585 100644 --- a/IPython/nbformat/v3/tests/test_json.py +++ b/IPython/nbformat/v3/tests/test_json.py @@ -1,4 +1,5 @@ import pprint +from base64 import decodestring from unittest import TestCase from ..nbjson import reads, writes @@ -29,5 +30,42 @@ class TestJSON(formattest.NBFormatTest, TestCase): s = writes(nb0, split_lines=True) self.assertEqual(nbjson.reads(s),nb0) + def test_read_png(self): + """PNG output data is b64 unicode""" + s = writes(nb0) + nb1 = nbjson.reads(s) + found_png = False + for cell in nb1.worksheets[0].cells: + if not 'outputs' in cell: + continue + for output in cell.outputs: + if 'png' in output: + found_png = True + pngdata = output['png'] + self.assertEqual(type(pngdata), unicode) + # test that it is valid b64 data + b64bytes = pngdata.encode('ascii') + raw_bytes = decodestring(b64bytes) + assert found_png, "never found png output" + + def test_read_jpeg(self): + """JPEG output data is b64 unicode""" + s = writes(nb0) + nb1 = nbjson.reads(s) + found_jpeg = False + for cell in nb1.worksheets[0].cells: + if not 'outputs' in cell: + continue + for output in cell.outputs: + if 'jpeg' in output: + found_jpeg = True + jpegdata = output['jpeg'] + self.assertEqual(type(jpegdata), unicode) + # test that it is valid b64 data + b64bytes = jpegdata.encode('ascii') + raw_bytes = decodestring(b64bytes) + assert found_jpeg, "never found jpeg output" + + diff --git a/IPython/nbformat/v3/tests/test_nbbase.py b/IPython/nbformat/v3/tests/test_nbbase.py index 67b4b79..2d137b8 100644 --- a/IPython/nbformat/v3/tests/test_nbbase.py +++ b/IPython/nbformat/v3/tests/test_nbbase.py @@ -141,3 +141,17 @@ class TestMetadata(TestCase): self.assertEqual(md.gistid, u'21341231') self.assertEqual(md.authors, authors) +class TestOutputs(TestCase): + def test_binary_png(self): + out = new_output(output_png=b'\x89PNG\r\n\x1a\n') + + def test_b64b6tes_png(self): + out = new_output(output_png=b'iVBORw0KG') + + def test_binary_jpeg(self): + out = new_output(output_jpeg=b'\xff\xd8') + + def test_b64b6tes_jpeg(self): + out = new_output(output_jpeg=b'/9') + + diff --git a/IPython/nbformat/v3/validator.py b/IPython/nbformat/v3/validator.py index b9b1244..3bbca3b 100755 --- a/IPython/nbformat/v3/validator.py +++ b/IPython/nbformat/v3/validator.py @@ -1,11 +1,11 @@ #!/usr/bin/env python # -*- coding: utf8 -*- +import argparse +import traceback +import json from IPython.external.jsonschema import Draft3Validator, validate, ValidationError import IPython.external.jsonpointer as jsonpointer -from IPython.external import argparse -import traceback -import json def nbvalidate(nbjson, schema='v3.withref.json', key=None,verbose=True): v3schema = resolve_ref(json.load(open(schema,'r'))) diff --git a/IPython/parallel/apps/baseapp.py b/IPython/parallel/apps/baseapp.py index 2f2033a..62346cd 100644 --- a/IPython/parallel/apps/baseapp.py +++ b/IPython/parallel/apps/baseapp.py @@ -71,7 +71,6 @@ class ParallelCrashHandler(CrashHandler): base_aliases = {} base_aliases.update(base_ip_aliases) base_aliases.update({ - 'profile-dir' : 'ProfileDir.location', 'work-dir' : 'BaseParallelApplication.work_dir', 'log-to-file' : 'BaseParallelApplication.log_to_file', 'clean-logs' : 'BaseParallelApplication.clean_logs', diff --git a/IPython/parallel/apps/ipcontrollerapp.py b/IPython/parallel/apps/ipcontrollerapp.py index 28c651c..a4b879d 100755 --- a/IPython/parallel/apps/ipcontrollerapp.py +++ b/IPython/parallel/apps/ipcontrollerapp.py @@ -11,7 +11,7 @@ Authors: """ #----------------------------------------------------------------------------- -# Copyright (C) 2008-2011 The IPython Development Team +# Copyright (C) 2008 The IPython Development Team # # Distributed under the terms of the BSD License. The full license is in # the file COPYING, distributed as part of this software. @@ -44,7 +44,7 @@ from IPython.parallel.apps.baseapp import ( catch_config_error, ) from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS +from IPython.utils.localinterfaces import localhost, public_ips from IPython.utils.traitlets import Instance, Unicode, Bool, List, Dict, TraitError from IPython.kernel.zmq.session import ( @@ -224,13 +224,13 @@ class IPControllerApp(BaseParallelApplication): location = cdict['location'] if not location: - if PUBLIC_IPS: - location = PUBLIC_IPS[-1] + if public_ips(): + location = public_ips()[-1] else: self.log.warn("Could not identify this machine's IP, assuming %s." " You may need to specify '--location=' to help" - " IPython decide when to connect via loopback." % LOCALHOST) - location = LOCALHOST + " IPython decide when to connect via loopback." % localhost() ) + location = localhost() cdict['location'] = location fname = os.path.join(self.profile_dir.security_dir, fname) self.log.info("writing connection info to %s", fname) diff --git a/IPython/parallel/apps/logwatcher.py b/IPython/parallel/apps/logwatcher.py index 80be410..c6ed6a3 100644 --- a/IPython/parallel/apps/logwatcher.py +++ b/IPython/parallel/apps/logwatcher.py @@ -26,7 +26,7 @@ import zmq from zmq.eventloop import ioloop, zmqstream from IPython.config.configurable import LoggingConfigurable -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import Int, Unicode, Instance, List #----------------------------------------------------------------------------- @@ -44,8 +44,10 @@ class LogWatcher(LoggingConfigurable): # configurables topics = List([''], config=True, help="The ZMQ topics to subscribe to. Default is to subscribe to all messages") - url = Unicode('tcp://%s:20202' % LOCALHOST, config=True, + url = Unicode(config=True, help="ZMQ url on which to listen for log messages") + def _url_default(self): + return 'tcp://%s:20202' % localhost() # internals stream = Instance('zmq.eventloop.zmqstream.ZMQStream') diff --git a/IPython/parallel/client/asyncresult.py b/IPython/parallel/client/asyncresult.py index aa58015..c26bbaf 100644 --- a/IPython/parallel/client/asyncresult.py +++ b/IPython/parallel/client/asyncresult.py @@ -391,7 +391,7 @@ class AsyncResult(object): tic = time.time() while not self.ready() and (timeout < 0 or time.time() - tic <= timeout): self.wait(interval) - clear_output() + clear_output(wait=True) print("%4i/%i tasks finished after %4i s" % (self.progress, N, self.elapsed), end="") sys.stdout.flush() print() diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 092272e..0b0c722 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -37,7 +37,7 @@ from IPython.core.profiledir import ProfileDir, ProfileDirError from IPython.utils.capture import RichOutput from IPython.utils.coloransi import TermColors from IPython.utils.jsonutil import rekey -from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS +from IPython.utils.localinterfaces import localhost, is_local_ip from IPython.utils.path import get_ipython_dir from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode, @@ -433,13 +433,13 @@ class Client(HasTraits): url = cfg['registration'] - if location is not None and addr == LOCALHOST: + if location is not None and addr == localhost(): # location specified, and connection is expected to be local - if location not in LOCAL_IPS and not sshserver: + if not is_local_ip(location) and not sshserver: # load ssh from JSON *only* if the controller is not on # this machine sshserver=cfg['ssh'] - if location not in LOCAL_IPS and not sshserver: + if not is_local_ip(location) and not sshserver: # warn if no ssh specified, but SSH is probably needed # This is only a warning, because the most likely cause # is a local Controller on a laptop whose IP is dynamic @@ -495,7 +495,12 @@ class Client(HasTraits): } self._queue_handlers = {'execute_reply' : self._handle_execute_reply, 'apply_reply' : self._handle_apply_reply} - self._connect(sshserver, ssh_kwargs, timeout) + + try: + self._connect(sshserver, ssh_kwargs, timeout) + except: + self.close(linger=0) + raise # last step: setup magics, if we are in IPython: @@ -599,7 +604,6 @@ class Client(HasTraits): self._connected=True def connect_socket(s, url): - # url = util.disambiguate_url(url, self._config['location']) if self._ssh: return tunnel.tunnel_connection(s, url, sshserver, **ssh_kwargs) else: @@ -956,14 +960,23 @@ class Client(HasTraits): view.activate(suffix) return view - def close(self): + def close(self, linger=None): + """Close my zmq Sockets + + If `linger`, set the zmq LINGER socket option, + which allows discarding of messages. + """ if self._closed: return self.stop_spin_thread() - snames = filter(lambda n: n.endswith('socket'), dir(self)) - for socket in map(lambda name: getattr(self, name), snames): - if isinstance(socket, zmq.Socket) and not socket.closed: - socket.close() + snames = [ trait for trait in self.trait_names() if trait.endswith("socket") ] + for name in snames: + socket = getattr(self, name) + if socket is not None and not socket.closed: + if linger is not None: + socket.close(linger=linger) + else: + socket.close() self._closed = True def _spin_every(self, interval=1): diff --git a/IPython/parallel/client/remotefunction.py b/IPython/parallel/client/remotefunction.py index 4b31a0b..026c0d0 100644 --- a/IPython/parallel/client/remotefunction.py +++ b/IPython/parallel/client/remotefunction.py @@ -89,7 +89,6 @@ def sync_view_results(f, self, *args, **kwargs): view = self.view if view._in_sync_results: return f(self, *args, **kwargs) - print 'in sync results', f view._in_sync_results = True try: ret = f(self, *args, **kwargs) diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index ae0b3a8..e049fc9 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -30,7 +30,7 @@ from zmq.eventloop.zmqstream import ZMQStream # internal: from IPython.utils.importstring import import_item -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import ( HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName @@ -177,20 +177,25 @@ class HubFactory(RegistrationFactory): def _notifier_port_default(self): return util.select_random_ports(1)[0] - engine_ip = Unicode(LOCALHOST, config=True, + engine_ip = Unicode(config=True, help="IP on which to listen for engine connections. [default: loopback]") + def _engine_ip_default(self): + return localhost() engine_transport = Unicode('tcp', config=True, help="0MQ transport for engine connections. [default: tcp]") - client_ip = Unicode(LOCALHOST, config=True, + client_ip = Unicode(config=True, help="IP on which to listen for client connections. [default: loopback]") client_transport = Unicode('tcp', config=True, help="0MQ transport for client connections. [default : tcp]") - monitor_ip = Unicode(LOCALHOST, config=True, + monitor_ip = Unicode(config=True, help="IP on which to listen for monitor messages. [default: loopback]") monitor_transport = Unicode('tcp', config=True, help="0MQ transport for monitor messages. [default : tcp]") + + _client_ip_default = _monitor_ip_default = _engine_ip_default + monitor_url = Unicode('') diff --git a/IPython/parallel/controller/mongodb.py b/IPython/parallel/controller/mongodb.py index 88d42e9..1d987e3 100644 --- a/IPython/parallel/controller/mongodb.py +++ b/IPython/parallel/controller/mongodb.py @@ -39,7 +39,7 @@ class MongoDB(BaseDB): necessary if the default mongodb configuration does not point to your mongod instance.""" ) - database = Unicode(config=True, + database = Unicode("ipython-tasks", config=True, help="""The MongoDB database name to use for storing tasks for this session. If unspecified, a new database will be created with the Hub's IDENT. Specifying the database will result in tasks from previous sessions being available via Clients' db_query and diff --git a/IPython/parallel/controller/scheduler.py b/IPython/parallel/controller/scheduler.py index 01fec8a..cc589fc 100644 --- a/IPython/parallel/controller/scheduler.py +++ b/IPython/parallel/controller/scheduler.py @@ -358,7 +358,7 @@ class TaskScheduler(SessionFactory): # build fake metadata md = dict( status=u'error', - engine=engine, + engine=engine.decode('ascii'), date=datetime.now(), ) msg = self.session.msg('apply_reply', content, parent=parent, metadata=md) diff --git a/IPython/parallel/controller/sqlitedb.py b/IPython/parallel/controller/sqlitedb.py index 36f015d..cf3c3e7 100644 --- a/IPython/parallel/controller/sqlitedb.py +++ b/IPython/parallel/controller/sqlitedb.py @@ -96,7 +96,7 @@ class SQLiteDB(BaseDB): location = Unicode('', config=True, help="""The directory containing the sqlite task database. The default is to use the cluster_dir location.""") - table = Unicode("", config=True, + table = Unicode("ipython-tasks", config=True, help="""The SQLite Table to use for storing tasks for this session. If unspecified, a new table will be created with the Hub's IDENT. Specifying the table will result in tasks from previous sessions being available via Clients' db_query and @@ -195,7 +195,7 @@ class SQLiteDB(BaseDB): If a bad (old) table does exist, return False """ - cursor = self._db.execute("PRAGMA table_info(%s)"%self.table) + cursor = self._db.execute("PRAGMA table_info('%s')"%self.table) lines = cursor.fetchall() if not lines: # table does not exist @@ -241,7 +241,7 @@ class SQLiteDB(BaseDB): ) previous_table = self.table - self._db.execute("""CREATE TABLE IF NOT EXISTS %s + self._db.execute("""CREATE TABLE IF NOT EXISTS '%s' (msg_id text PRIMARY KEY, header dict text, metadata dict text, @@ -333,12 +333,12 @@ class SQLiteDB(BaseDB): d['msg_id'] = msg_id line = self._dict_to_list(d) tups = '(%s)'%(','.join(['?']*len(line))) - self._db.execute("INSERT INTO %s VALUES %s"%(self.table, tups), line) + self._db.execute("INSERT INTO '%s' VALUES %s"%(self.table, tups), line) # self._db.commit() def get_record(self, msg_id): """Get a specific Task Record, by msg_id.""" - cursor = self._db.execute("""SELECT * FROM %s WHERE msg_id==?"""%self.table, (msg_id,)) + cursor = self._db.execute("""SELECT * FROM '%s' WHERE msg_id==?"""%self.table, (msg_id,)) line = cursor.fetchone() if line is None: raise KeyError("No such msg: %r"%msg_id) @@ -346,7 +346,7 @@ class SQLiteDB(BaseDB): def update_record(self, msg_id, rec): """Update the data in an existing record.""" - query = "UPDATE %s SET "%self.table + query = "UPDATE '%s' SET "%self.table sets = [] keys = sorted(rec.keys()) values = [] @@ -361,13 +361,13 @@ class SQLiteDB(BaseDB): def drop_record(self, msg_id): """Remove a record from the DB.""" - self._db.execute("""DELETE FROM %s WHERE msg_id==?"""%self.table, (msg_id,)) + self._db.execute("""DELETE FROM '%s' WHERE msg_id==?"""%self.table, (msg_id,)) # self._db.commit() def drop_matching_records(self, check): """Remove a record from the DB.""" expr,args = self._render_expression(check) - query = "DELETE FROM %s WHERE %s"%(self.table, expr) + query = "DELETE FROM '%s' WHERE %s"%(self.table, expr) self._db.execute(query,args) # self._db.commit() @@ -399,7 +399,7 @@ class SQLiteDB(BaseDB): else: req = '*' expr,args = self._render_expression(check) - query = """SELECT %s FROM %s WHERE %s"""%(req, self.table, expr) + query = """SELECT %s FROM '%s' WHERE %s"""%(req, self.table, expr) cursor = self._db.execute(query, args) matches = cursor.fetchall() records = [] @@ -410,7 +410,7 @@ class SQLiteDB(BaseDB): def get_history(self): """get all msg_ids, ordered by time submitted.""" - query = """SELECT msg_id FROM %s ORDER by submitted ASC"""%self.table + query = """SELECT msg_id FROM '%s' ORDER by submitted ASC"""%self.table cursor = self._db.execute(query) # will be a list of length 1 tuples return [ tup[0] for tup in cursor.fetchall()] diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index 208212f..e60f2bf 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -24,7 +24,7 @@ from zmq.eventloop import ioloop, zmqstream from IPython.external.ssh import tunnel # internal -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import ( Instance, Dict, Integer, Type, Float, Integer, Unicode, CBytes, Bool ) @@ -184,13 +184,13 @@ class EngineFactory(RegistrationFactory): if self.max_heartbeat_misses > 0: # Add a monitor socket which will record the last time a ping was seen mon = self.context.socket(zmq.SUB) - mport = mon.bind_to_random_port('tcp://%s' % LOCALHOST) + mport = mon.bind_to_random_port('tcp://%s' % localhost()) mon.setsockopt(zmq.SUBSCRIBE, b"") self._hb_listener = zmqstream.ZMQStream(mon, self.loop) self._hb_listener.on_recv(self._report_ping) - hb_monitor = "tcp://%s:%i" % (LOCALHOST, mport) + hb_monitor = "tcp://%s:%i" % (localhost(), mport) heart = Heart(hb_ping, hb_pong, hb_monitor , heart_id=identity) heart.start() diff --git a/IPython/parallel/factory.py b/IPython/parallel/factory.py index 3b2a770..4636136 100644 --- a/IPython/parallel/factory.py +++ b/IPython/parallel/factory.py @@ -24,7 +24,7 @@ import zmq from zmq.eventloop.ioloop import IOLoop from IPython.config.configurable import Configurable -from IPython.utils.localinterfaces import LOCALHOST +from IPython.utils.localinterfaces import localhost from IPython.utils.traitlets import Integer, Instance, Unicode from IPython.parallel.util import select_random_ports @@ -40,16 +40,18 @@ class RegistrationFactory(SessionFactory): url = Unicode('', config=True, help="""The 0MQ url used for registration. This sets transport, ip, and port - in one variable. For example: url='tcp://%s:12345' or + in one variable. For example: url='tcp://127.0.0.1:12345' or url='epgm://*:90210'""" - % LOCALHOST) # url takes precedence over ip,regport,transport + ) # url takes precedence over ip,regport,transport transport = Unicode('tcp', config=True, help="""The 0MQ transport for communications. This will likely be the default of 'tcp', but other values include 'ipc', 'epgm', 'inproc'.""") - ip = Unicode(LOCALHOST, config=True, + ip = Unicode(config=True, help="""The IP address for registration. This is generally either '127.0.0.1' for loopback only or '*' for all interfaces. - [default: '%s']""" % LOCALHOST) + """) + def _ip_default(self): + return localhost() regport = Integer(config=True, help="""The port on which the Hub listens for registration.""") def _regport_default(self): diff --git a/IPython/parallel/tests/__init__.py b/IPython/parallel/tests/__init__.py index a1271c0..6e456cc 100644 --- a/IPython/parallel/tests/__init__.py +++ b/IPython/parallel/tests/__init__.py @@ -63,8 +63,7 @@ def setup(): tic = time.time() while not os.path.exists(engine_json) or not os.path.exists(client_json): if cp.poll() is not None: - print cp.poll() - raise RuntimeError("The test controller failed to start.") + raise RuntimeError("The test controller exited with status %s" % cp.poll()) elif time.time()-tic > 15: raise RuntimeError("Timeout waiting for the test controller to start.") time.sleep(0.1) diff --git a/IPython/parallel/tests/test_client.py b/IPython/parallel/tests/test_client.py index 2464755..24ebd14 100644 --- a/IPython/parallel/tests/test_client.py +++ b/IPython/parallel/tests/test_client.py @@ -172,7 +172,6 @@ class TestClient(ClusterTestCase): # give the monitor time to notice the message time.sleep(.25) ahr = self.client.get_result(ar.msg_ids[0]) - print ar.get(), ahr.get(), ar._single_result, ahr._single_result self.assertTrue(isinstance(ahr, AsyncHubResult)) self.assertEqual(ahr.get().pyout, ar.get().pyout) ar2 = self.client.get_result(ar.msg_ids[0]) diff --git a/IPython/parallel/tests/test_magics.py b/IPython/parallel/tests/test_magics.py index 102e40c..c8ec444 100644 --- a/IPython/parallel/tests/test_magics.py +++ b/IPython/parallel/tests/test_magics.py @@ -17,20 +17,14 @@ Authors: #------------------------------------------------------------------------------- import re -import sys import time -import zmq -from nose import SkipTest from IPython.testing import decorators as dec -from IPython.testing.ipunittest import ParametricTestCase from IPython.utils.io import capture_output from IPython import parallel as pmod -from IPython.parallel import error from IPython.parallel import AsyncResult -from IPython.parallel.util import interactive from IPython.parallel.tests import add_engines @@ -39,7 +33,7 @@ from .clienttest import ClusterTestCase, generate_output def setup(): add_engines(3, total=True) -class TestParallelMagics(ClusterTestCase, ParametricTestCase): +class TestParallelMagics(ClusterTestCase): def test_px_blocking(self): ip = get_ipython() diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 40b0353..3cc704e 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -22,20 +22,16 @@ import platform import time from collections import namedtuple from tempfile import mktemp -from StringIO import StringIO import zmq -from nose import SkipTest from nose.plugins.attrib import attr from IPython.testing import decorators as dec -from IPython.testing.ipunittest import ParametricTestCase from IPython.utils.io import capture_output from IPython import parallel as pmod from IPython.parallel import error from IPython.parallel import AsyncResult, AsyncHubResult, AsyncMapResult -from IPython.parallel import DirectView from IPython.parallel.util import interactive from IPython.parallel.tests import add_engines @@ -47,7 +43,7 @@ def setup(): point = namedtuple("point", "x y") -class TestView(ClusterTestCase, ParametricTestCase): +class TestView(ClusterTestCase): def setUp(self): # On Win XP, wait for resource cleanup, else parallel test group fails @@ -242,7 +238,7 @@ class TestView(ClusterTestCase, ParametricTestCase): @skip_without('numpy') def test_scatter_gather_numpy(self): import numpy - from numpy.testing.utils import assert_array_equal, assert_array_almost_equal + from numpy.testing.utils import assert_array_equal view = self.client[:] a = numpy.arange(64) view.scatter('a', a, block=True) @@ -280,7 +276,7 @@ class TestView(ClusterTestCase, ParametricTestCase): def test_apply_numpy(self): """view.apply(f, ndarray)""" import numpy - from numpy.testing.utils import assert_array_equal, assert_array_almost_equal + from numpy.testing.utils import assert_array_equal A = numpy.random.random((100,100)) view = self.client[-1] @@ -368,7 +364,7 @@ class TestView(ClusterTestCase, ParametricTestCase): @skip_without('numpy') def test_scatter_gather_numpy_nonblocking(self): import numpy - from numpy.testing.utils import assert_array_equal, assert_array_almost_equal + from numpy.testing.utils import assert_array_equal a = numpy.arange(64) view = self.client[:] ar = view.scatter('a', a, block=False) @@ -512,19 +508,17 @@ class TestView(ClusterTestCase, ParametricTestCase): def test_len(self): """len(view) makes sense""" e0 = self.client[self.client.ids[0]] - yield self.assertEqual(len(e0), 1) + self.assertEqual(len(e0), 1) v = self.client[:] - yield self.assertEqual(len(v), len(self.client.ids)) + self.assertEqual(len(v), len(self.client.ids)) v = self.client.direct_view('all') - yield self.assertEqual(len(v), len(self.client.ids)) + self.assertEqual(len(v), len(self.client.ids)) v = self.client[:2] - yield self.assertEqual(len(v), 2) + self.assertEqual(len(v), 2) v = self.client[:1] - yield self.assertEqual(len(v), 1) + self.assertEqual(len(v), 1) v = self.client.load_balanced_view() - yield self.assertEqual(len(v), len(self.client.ids)) - # parametric tests seem to require manual closing? - self.client.close() + self.assertEqual(len(v), len(self.client.ids)) # begin execute tests diff --git a/IPython/parallel/util.py b/IPython/parallel/util.py index a725d5a..cf0824a 100644 --- a/IPython/parallel/util.py +++ b/IPython/parallel/util.py @@ -43,7 +43,7 @@ from IPython.external.decorator import decorator # IPython imports from IPython.config.application import Application -from IPython.utils.localinterfaces import LOCALHOST, PUBLIC_IPS +from IPython.utils.localinterfaces import localhost, is_public_ip, public_ips from IPython.kernel.zmq.log import EnginePUBHandler from IPython.kernel.zmq.serialize import ( unserialize_object, serialize_object, pack_apply_message, unpack_apply_message @@ -187,9 +187,9 @@ def disambiguate_ip_address(ip, location=None): """turn multi-ip interfaces '0.0.0.0' and '*' into connectable ones, based on the location (default interpretation of location is localhost).""" if ip in ('0.0.0.0', '*'): - if location is None or location in PUBLIC_IPS or not PUBLIC_IPS: + if location is None or is_public_ip(location) or not public_ips(): # If location is unspecified or cannot be determined, assume local - ip = LOCALHOST + ip = localhost() elif location: return location return ip diff --git a/IPython/qt/console/qtconsoleapp.py b/IPython/qt/console/qtconsoleapp.py index e8ddbf4..3ef7fa1 100644 --- a/IPython/qt/console/qtconsoleapp.py +++ b/IPython/qt/console/qtconsoleapp.py @@ -74,7 +74,7 @@ from IPython.consoleapp import ( # Network Constants #----------------------------------------------------------------------------- -from IPython.utils.localinterfaces import LOCALHOST, LOCAL_IPS +from IPython.utils.localinterfaces import is_local_ip #----------------------------------------------------------------------------- # Globals @@ -250,7 +250,7 @@ class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp): QtGui.QApplication.setWindowIcon(self.app.icon) ip = self.ip - local_kernel = (not self.existing) or ip in LOCAL_IPS + local_kernel = (not self.existing) or is_local_ip(ip) self.widget = self.widget_factory(config=self.config, local_kernel=local_kernel) self.init_colors(self.widget) diff --git a/IPython/qt/console/tests/test_app.py b/IPython/qt/console/tests/test_app.py new file mode 100644 index 0000000..d28596e --- /dev/null +++ b/IPython/qt/console/tests/test_app.py @@ -0,0 +1,25 @@ +"""Test QtConsoleApp""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2013 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import nose.tools as nt + +import IPython.testing.tools as tt + +#----------------------------------------------------------------------------- +# Test functions +#----------------------------------------------------------------------------- + +def test_help_output(): + """ipython qtconsole --help-all works""" + tt.help_all_output_test('qtconsole') + diff --git a/IPython/scripts/iptest b/IPython/scripts/iptest index 9d4b6da..b6d523c 100755 --- a/IPython/scripts/iptest +++ b/IPython/scripts/iptest @@ -20,5 +20,5 @@ Exiting.""" import sys print >> sys.stderr, error else: - from IPython.testing import iptest - iptest.main() + from IPython.testing import iptestcontroller + iptestcontroller.main() diff --git a/IPython/terminal/console/tests/test_console.py b/IPython/terminal/console/tests/test_console.py index 60b9679..09351a0 100644 --- a/IPython/terminal/console/tests/test_console.py +++ b/IPython/terminal/console/tests/test_console.py @@ -18,11 +18,12 @@ import time import nose.tools as nt from nose import SkipTest +import IPython.testing.tools as tt from IPython.testing import decorators as dec from IPython.utils import py3compat #----------------------------------------------------------------------------- -# Test functions begin +# Tests #----------------------------------------------------------------------------- @dec.skip_win32 @@ -55,3 +56,8 @@ def test_console_starts(): p.expect([pexpect.EOF, pexpect.TIMEOUT], timeout=t) if p.isalive(): p.terminate() + +def test_help_output(): + """ipython console --help-all works""" + tt.help_all_output_test('console') + diff --git a/IPython/terminal/embed.py b/IPython/terminal/embed.py index 2a956b0..d343955 100644 --- a/IPython/terminal/embed.py +++ b/IPython/terminal/embed.py @@ -29,6 +29,7 @@ import warnings from IPython.core import ultratb, compilerop from IPython.core.magic import Magics, magics_class, line_magic +from IPython.core.interactiveshell import DummyMod from IPython.terminal.interactiveshell import TerminalInteractiveShell from IPython.terminal.ipapp import load_default_config @@ -185,9 +186,6 @@ class InteractiveShellEmbed(TerminalInteractiveShell): there is no fundamental reason why it can't work perfectly.""" if (global_ns is not None) and (module is None): - class DummyMod(object): - """A dummy module object for embedded IPython.""" - pass warnings.warn("global_ns is deprecated, use module instead.", DeprecationWarning) module = DummyMod() module.__dict__ = global_ns diff --git a/IPython/terminal/interactiveshell.py b/IPython/terminal/interactiveshell.py index 3a7c93a..5b034d0 100644 --- a/IPython/terminal/interactiveshell.py +++ b/IPython/terminal/interactiveshell.py @@ -369,7 +369,7 @@ class TerminalInteractiveShell(InteractiveShell): for name, cmd in aliases: - self.alias_manager.define_alias(name, cmd) + self.alias_manager.soft_define_alias(name, cmd) #------------------------------------------------------------------------- # Things related to the banner and usage diff --git a/IPython/terminal/ipapp.py b/IPython/terminal/ipapp.py index 02ba278..3a0c314 100755 --- a/IPython/terminal/ipapp.py +++ b/IPython/terminal/ipapp.py @@ -47,6 +47,7 @@ from IPython.core.magics import ScriptMagics from IPython.core.shellapp import ( InteractiveShellApp, shell_flags, shell_aliases ) +from IPython.extensions.storemagic import StoreMagics from IPython.terminal.interactiveshell import TerminalInteractiveShell from IPython.utils import warn from IPython.utils.path import get_ipython_dir, check_for_old_config @@ -219,6 +220,7 @@ class TerminalIPythonApp(BaseIPythonApplication, InteractiveShellApp): PlainTextFormatter, IPCompleter, ScriptMagics, + StoreMagics, ] subcommands = Dict(dict( diff --git a/IPython/terminal/tests/test_help.py b/IPython/terminal/tests/test_help.py new file mode 100644 index 0000000..1c45093 --- /dev/null +++ b/IPython/terminal/tests/test_help.py @@ -0,0 +1,37 @@ +"""Test help output of various IPython entry points""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2013 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import IPython.testing.tools as tt + +#----------------------------------------------------------------------------- +# Tests +#----------------------------------------------------------------------------- + + +def test_ipython_help(): + tt.help_all_output_test() + +def test_profile_help(): + tt.help_all_output_test("profile") + +def test_profile_list_help(): + tt.help_all_output_test("profile list") + +def test_profile_create_help(): + tt.help_all_output_test("profile create") + +def test_locate_help(): + tt.help_all_output_test("locate") + +def test_locate_profile_help(): + tt.help_all_output_test("locate profile") diff --git a/IPython/testing/_paramtestpy2.py b/IPython/testing/_paramtestpy2.py deleted file mode 100644 index 102bc85..0000000 --- a/IPython/testing/_paramtestpy2.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Implementation of the parametric test support for Python 2.x -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2009-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -import sys -import unittest -from compiler.consts import CO_GENERATOR - -#----------------------------------------------------------------------------- -# Classes and functions -#----------------------------------------------------------------------------- - -def isgenerator(func): - try: - return func.func_code.co_flags & CO_GENERATOR != 0 - except AttributeError: - return False - -class ParametricTestCase(unittest.TestCase): - """Write parametric tests in normal unittest testcase form. - - Limitations: the last iteration misses printing out a newline when running - in verbose mode. - """ - def run_parametric(self, result, testMethod): - # But if we have a test generator, we iterate it ourselves - testgen = testMethod() - while True: - try: - # Initialize test - result.startTest(self) - - # SetUp - try: - self.setUp() - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - return - # Test execution - ok = False - try: - next(testgen) - ok = True - except StopIteration: - # We stop the loop - break - except self.failureException: - result.addFailure(self, sys.exc_info()) - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - # TearDown - try: - self.tearDown() - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - ok = False - if ok: result.addSuccess(self) - - finally: - result.stopTest(self) - - def run(self, result=None): - if result is None: - result = self.defaultTestResult() - testMethod = getattr(self, self._testMethodName) - # For normal tests, we just call the base class and return that - if isgenerator(testMethod): - return self.run_parametric(result, testMethod) - else: - return super(ParametricTestCase, self).run(result) - - -def parametric(func): - """Decorator to make a simple function into a normal test via unittest.""" - - class Tester(ParametricTestCase): - test = staticmethod(func) - - Tester.__name__ = func.__name__ - - return Tester diff --git a/IPython/testing/_paramtestpy3.py b/IPython/testing/_paramtestpy3.py deleted file mode 100644 index 5956b5f..0000000 --- a/IPython/testing/_paramtestpy3.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Implementation of the parametric test support for Python 3.x. - -Thanks for the py3 version to Robert Collins, from the Testing in Python -mailing list. -""" - -#----------------------------------------------------------------------------- -# Copyright (C) 2009-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -import unittest -from unittest import TestSuite - -#----------------------------------------------------------------------------- -# Classes and functions -#----------------------------------------------------------------------------- - - -def isgenerator(func): - return hasattr(func,'_generator') - - -class IterCallableSuite(TestSuite): - def __init__(self, iterator, adapter): - self._iter = iterator - self._adapter = adapter - def __iter__(self): - yield self._adapter(self._iter.__next__) - -class ParametricTestCase(unittest.TestCase): - """Write parametric tests in normal unittest testcase form. - - Limitations: the last iteration misses printing out a newline when - running in verbose mode. - """ - - def run(self, result=None): - testMethod = getattr(self, self._testMethodName) - # For normal tests, we just call the base class and return that - if isgenerator(testMethod): - def adapter(next_test): - ftc = unittest.FunctionTestCase(next_test, - self.setUp, - self.tearDown) - self._nose_case = ftc # Nose 1.0 rejects the test without this - return ftc - - return IterCallableSuite(testMethod(),adapter).run(result) - else: - return super(ParametricTestCase, self).run(result) - - -def parametric(func): - """Decorator to make a simple function into a normal test via -unittest.""" - # Hack, until I figure out how to write isgenerator() for python3!! - func._generator = True - - class Tester(ParametricTestCase): - test = staticmethod(func) - - Tester.__name__ = func.__name__ - - return Tester diff --git a/IPython/testing/decorators.py b/IPython/testing/decorators.py index 8763002..3ad0548 100644 --- a/IPython/testing/decorators.py +++ b/IPython/testing/decorators.py @@ -16,10 +16,6 @@ Included decorators: Lightweight testing that remains unittest-compatible. -- @parametric, for parametric test support that is vastly easier to use than - nose's for debugging. With ours, if a test fails, the stack under inspection - is that of the test and not that of the test framework. - - An @as_unittest decorator can be used to tag any normal parameter-less function as a unittest TestCase. Then, both nose and normal unittest will recognize it as such. This will make it easier to migrate away from Nose if @@ -58,12 +54,6 @@ import unittest # This is Michele Simionato's decorator module, kept verbatim. from IPython.external.decorator import decorator -# We already have python3-compliant code for parametric tests -if sys.version[0]=='2': - from _paramtestpy2 import parametric -else: - from _paramtestpy3 import parametric - # Expose the unittest-driven decorators from ipunittest import ipdoctest, ipdocstring @@ -384,7 +374,7 @@ def onlyif_cmds_exist(*commands): "is installed".format(cmd)) except ImportError as e: # is_cmd_found uses pywin32 on windows, which might not be available - if sys.platform == 'win32' and 'pywin32' in e.message: + if sys.platform == 'win32' and 'pywin32' in str(e): return skip("This test runs only if pywin32 and command '{0}' " "is installed".format(cmd)) raise e diff --git a/IPython/testing/iptest.py b/IPython/testing/iptest.py index b4aba1c..054574b 100644 --- a/IPython/testing/iptest.py +++ b/IPython/testing/iptest.py @@ -30,11 +30,8 @@ from __future__ import print_function import glob import os import os.path as path -import signal +import re import sys -import subprocess -import tempfile -import time import warnings # Now, proceed to import nose itself @@ -42,17 +39,10 @@ import nose.plugins.builtin from nose.plugins.xunit import Xunit from nose import SkipTest from nose.core import TestProgram +from nose.plugins import Plugin # Our own imports -from IPython.utils import py3compat from IPython.utils.importstring import import_item -from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir -from IPython.utils.process import pycmd2argv -from IPython.utils.sysinfo import sys_info -from IPython.utils.tempdir import TemporaryDirectory -from IPython.utils.warn import warn - -from IPython.testing import globalipapp from IPython.testing.plugin.ipdoctest import IPythonDoctest from IPython.external.decorators import KnownFailure, knownfailureif @@ -99,7 +89,7 @@ def monkeypatch_xunit(): Xunit.addError = addError #----------------------------------------------------------------------------- -# Logic for skipping doctests +# Check which dependencies are installed and greater than minimum version. #----------------------------------------------------------------------------- def extract_version(mod): return mod.__version__ @@ -163,314 +153,204 @@ min_zmq = (2,1,11) have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x()) #----------------------------------------------------------------------------- -# Functions and classes +# Test suite definitions #----------------------------------------------------------------------------- -def report(): - """Return a string with a summary report of test-related variables.""" - - out = [ sys_info(), '\n'] - - avail = [] - not_avail = [] - - for k, is_avail in have.items(): - if is_avail: - avail.append(k) - else: - not_avail.append(k) - - if avail: - out.append('\nTools and libraries available at test time:\n') - avail.sort() - out.append(' ' + ' '.join(avail)+'\n') - - if not_avail: - out.append('\nTools and libraries NOT available at test time:\n') - not_avail.sort() - out.append(' ' + ' '.join(not_avail)+'\n') - - return ''.join(out) - - -def make_exclude(): - """Make patterns of modules and packages to exclude from testing. - - For the IPythonDoctest plugin, we need to exclude certain patterns that - cause testing problems. We should strive to minimize the number of - skipped modules, since this means untested code. - - These modules and packages will NOT get scanned by nose at all for tests. - """ - # Simple utility to make IPython paths more readably, we need a lot of - # these below - ipjoin = lambda *paths: pjoin('IPython', *paths) - - exclusions = [ipjoin('external'), - ipjoin('quarantine'), - ipjoin('deathrow'), - # This guy is probably attic material - ipjoin('testing', 'mkdoctests'), - # Testing inputhook will need a lot of thought, to figure out - # how to have tests that don't lock up with the gui event - # loops in the picture - ipjoin('lib', 'inputhook'), - # Config files aren't really importable stand-alone - ipjoin('config', 'profile'), - # The notebook 'static' directory contains JS, css and other - # files for web serving. Occasionally projects may put a .py - # file in there (MathJax ships a conf.py), so we might as - # well play it safe and skip the whole thing. - ipjoin('html', 'static'), - ipjoin('html', 'fabfile'), - ] - if not have['sqlite3']: - exclusions.append(ipjoin('core', 'tests', 'test_history')) - exclusions.append(ipjoin('core', 'history')) - if not have['wx']: - exclusions.append(ipjoin('lib', 'inputhookwx')) +test_group_names = ['parallel', 'kernel', 'kernel.inprocess', 'config', 'core', + 'extensions', 'lib', 'terminal', 'testing', 'utils', + 'nbformat', 'qt', 'html', 'nbconvert' + ] + +class TestSection(object): + def __init__(self, name, includes): + self.name = name + self.includes = includes + self.excludes = [] + self.dependencies = [] + self.enabled = True - if 'IPython.kernel.inprocess' not in sys.argv: - exclusions.append(ipjoin('kernel', 'inprocess')) + def exclude(self, module): + if not module.startswith('IPython'): + module = self.includes[0] + "." + module + self.excludes.append(module.replace('.', os.sep)) - # FIXME: temporarily disable autoreload tests, as they can produce - # spurious failures in subsequent tests (cythonmagic). - exclusions.append(ipjoin('extensions', 'autoreload')) - exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload')) - - # We do this unconditionally, so that the test suite doesn't import - # gtk, changing the default encoding and masking some unicode bugs. - exclusions.append(ipjoin('lib', 'inputhookgtk')) - exclusions.append(ipjoin('kernel', 'zmq', 'gui', 'gtkembed')) - - #Also done unconditionally, exclude nbconvert directories containing - #config files used to test. Executing the config files with iptest would - #cause an exception. - exclusions.append(ipjoin('nbconvert', 'tests', 'files')) - exclusions.append(ipjoin('nbconvert', 'exporters', 'tests', 'files')) - - # These have to be skipped on win32 because the use echo, rm, cd, etc. - # See ticket https://github.com/ipython/ipython/issues/87 - if sys.platform == 'win32': - exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip')) - exclusions.append(ipjoin('testing', 'plugin', 'dtexample')) - - if not have['pexpect']: - exclusions.extend([ipjoin('lib', 'irunner'), - ipjoin('lib', 'tests', 'test_irunner'), - ipjoin('terminal', 'console'), - ]) - - if not have['zmq']: - exclusions.append(ipjoin('lib', 'kernel')) - exclusions.append(ipjoin('kernel')) - exclusions.append(ipjoin('qt')) - exclusions.append(ipjoin('html')) - exclusions.append(ipjoin('consoleapp.py')) - exclusions.append(ipjoin('terminal', 'console')) - exclusions.append(ipjoin('parallel')) - elif not have['qt'] or not have['pygments']: - exclusions.append(ipjoin('qt')) - - if not have['pymongo']: - exclusions.append(ipjoin('parallel', 'controller', 'mongodb')) - exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb')) - - if not have['matplotlib']: - exclusions.extend([ipjoin('core', 'pylabtools'), - ipjoin('core', 'tests', 'test_pylabtools'), - ipjoin('kernel', 'zmq', 'pylab'), - ]) - - if not have['cython']: - exclusions.extend([ipjoin('extensions', 'cythonmagic')]) - exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')]) - - if not have['oct2py']: - exclusions.extend([ipjoin('extensions', 'octavemagic')]) - exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')]) - - if not have['tornado']: - exclusions.append(ipjoin('html')) - - if not have['jinja2']: - exclusions.append(ipjoin('html', 'notebookapp')) - - if not have['rpy2'] or not have['numpy']: - exclusions.append(ipjoin('extensions', 'rmagic')) - exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic')) - - if not have['azure']: - exclusions.append(ipjoin('html', 'services', 'notebooks', 'azurenbmanager')) - - if not all((have['pygments'], have['jinja2'], have['sphinx'])): - exclusions.append(ipjoin('nbconvert')) - - # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. - if sys.platform == 'win32': - exclusions = [s.replace('\\','\\\\') for s in exclusions] + def requires(self, *packages): + self.dependencies.extend(packages) - # check for any exclusions that don't seem to exist: - parent, _ = os.path.split(get_ipython_package_dir()) - for exclusion in exclusions: - if exclusion.endswith(('deathrow', 'quarantine')): - # ignore deathrow/quarantine, which exist in dev, but not install - continue - fullpath = pjoin(parent, exclusion) - if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'): - warn("Excluding nonexistent file: %r" % exclusion) - - return exclusions - - -class IPTester(object): - """Call that calls iptest or trial in a subprocess. - """ - #: string, name of test runner that will be called - runner = None - #: list, parameters for test runner - params = None - #: list, arguments of system call to be made to call test runner - call_args = None - #: list, subprocesses we start (for cleanup) - processes = None - #: str, coverage xml output file - coverage_xml = None - - def __init__(self, runner='iptest', params=None): - """Create new test runner.""" - p = os.path - if runner == 'iptest': - iptest_app = os.path.abspath(get_ipython_module_path('IPython.testing.iptest')) - self.runner = pycmd2argv(iptest_app) + sys.argv[1:] - else: - raise Exception('Not a valid test runner: %s' % repr(runner)) - if params is None: - params = [] - if isinstance(params, str): - params = [params] - self.params = params - - # Assemble call - self.call_args = self.runner+self.params - - # Find the section we're testing (IPython.foo) - for sect in self.params: - if sect.startswith('IPython') or sect in special_test_suites: break - else: - raise ValueError("Section not found", self.params) - - if '--with-xunit' in self.call_args: - - self.call_args.append('--xunit-file') - # FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary: - xunit_file = path.abspath(sect+'.xunit.xml') - if sys.platform == 'win32': - xunit_file = '"%s"' % xunit_file - self.call_args.append(xunit_file) - - if '--with-xml-coverage' in self.call_args: - self.coverage_xml = path.abspath(sect+".coverage.xml") - self.call_args.remove('--with-xml-coverage') - self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:] - - # Store anything we start to clean up on deletion - self.processes = [] - - def _run_cmd(self): - with TemporaryDirectory() as IPYTHONDIR: - env = os.environ.copy() - env['IPYTHONDIR'] = IPYTHONDIR - # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg - subp = subprocess.Popen(self.call_args, env=env) - self.processes.append(subp) - # If this fails, the process will be left in self.processes and - # cleaned up later, but if the wait call succeeds, then we can - # clear the stored process. - retcode = subp.wait() - self.processes.pop() - return retcode - - def run(self): - """Run the stored commands""" - try: - retcode = self._run_cmd() - except KeyboardInterrupt: - return -signal.SIGINT - except: - import traceback - traceback.print_exc() - return 1 # signal failure - - if self.coverage_xml: - subprocess.call(["coverage", "xml", "-o", self.coverage_xml]) - return retcode - - def __del__(self): - """Cleanup on exit by killing any leftover processes.""" - for subp in self.processes: - if subp.poll() is not None: - continue # process is already dead - - try: - print('Cleaning up stale PID: %d' % subp.pid) - subp.kill() - except: # (OSError, WindowsError) ? - # This is just a best effort, if we fail or the process was - # really gone, ignore it. - pass - else: - for i in range(10): - if subp.poll() is None: - time.sleep(0.1) - else: - break - - if subp.poll() is None: - # The process did not die... - print('... failed. Manual cleanup may be required.') - - -special_test_suites = { - 'autoreload': ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'], -} - -def make_runners(inc_slow=False): - """Define the top-level packages that need to be tested. - """ - - # Packages to be tested via nose, that only depend on the stdlib - nose_pkg_names = ['config', 'core', 'extensions', 'lib', 'terminal', - 'testing', 'utils', 'nbformat'] + @property + def will_run(self): + return self.enabled and all(have[p] for p in self.dependencies) + +# Name -> (include, exclude, dependencies_met) +test_sections = {n:TestSection(n, ['IPython.%s' % n]) for n in test_group_names} + +# Exclusions and dependencies +# --------------------------- + +# core: +sec = test_sections['core'] +if not have['sqlite3']: + sec.exclude('tests.test_history') + sec.exclude('history') +if not have['matplotlib']: + sec.exclude('pylabtools'), + sec.exclude('tests.test_pylabtools') + +# lib: +sec = test_sections['lib'] +if not have['wx']: + sec.exclude('inputhookwx') +if not have['pexpect']: + sec.exclude('irunner') + sec.exclude('tests.test_irunner') +if not have['zmq']: + sec.exclude('kernel') +# We do this unconditionally, so that the test suite doesn't import +# gtk, changing the default encoding and masking some unicode bugs. +sec.exclude('inputhookgtk') +# Testing inputhook will need a lot of thought, to figure out +# how to have tests that don't lock up with the gui event +# loops in the picture +sec.exclude('inputhook') + +# testing: +sec = test_sections['testing'] +# This guy is probably attic material +sec.exclude('mkdoctests') +# These have to be skipped on win32 because they use echo, rm, cd, etc. +# See ticket https://github.com/ipython/ipython/issues/87 +if sys.platform == 'win32': + sec.exclude('plugin.test_exampleip') + sec.exclude('plugin.dtexample') + +# terminal: +if (not have['pexpect']) or (not have['zmq']): + test_sections['terminal'].exclude('console') + +# parallel +sec = test_sections['parallel'] +sec.requires('zmq') +if not have['pymongo']: + sec.exclude('controller.mongodb') + sec.exclude('tests.test_mongodb') + +# kernel: +sec = test_sections['kernel'] +sec.requires('zmq') +# The in-process kernel tests are done in a separate section +sec.exclude('inprocess') +# importing gtk sets the default encoding, which we want to avoid +sec.exclude('zmq.gui.gtkembed') +if not have['matplotlib']: + sec.exclude('zmq.pylab') + +# kernel.inprocess: +test_sections['kernel.inprocess'].requires('zmq') + +# extensions: +sec = test_sections['extensions'] +if not have['cython']: + sec.exclude('cythonmagic') + sec.exclude('tests.test_cythonmagic') +if not have['oct2py']: + sec.exclude('octavemagic') + sec.exclude('tests.test_octavemagic') +if not have['rpy2'] or not have['numpy']: + sec.exclude('rmagic') + sec.exclude('tests.test_rmagic') +# autoreload does some strange stuff, so move it to its own test section +sec.exclude('autoreload') +sec.exclude('tests.test_autoreload') +test_sections['autoreload'] = TestSection('autoreload', + ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload']) +test_group_names.append('autoreload') + +# qt: +test_sections['qt'].requires('zmq', 'qt', 'pygments') + +# html: +sec = test_sections['html'] +sec.requires('zmq', 'tornado') +# The notebook 'static' directory contains JS, css and other +# files for web serving. Occasionally projects may put a .py +# file in there (MathJax ships a conf.py), so we might as +# well play it safe and skip the whole thing. +sec.exclude('static') +sec.exclude('fabfile') +if not have['jinja2']: + sec.exclude('notebookapp') +if not have['azure']: + sec.exclude('services.notebooks.azurenbmanager') + +# config: +# Config files aren't really importable stand-alone +test_sections['config'].exclude('profile') + +# nbconvert: +sec = test_sections['nbconvert'] +sec.requires('pygments', 'jinja2', 'sphinx') +# Exclude nbconvert directories containing config files used to test. +# Executing the config files with iptest would cause an exception. +sec.exclude('tests.files') +sec.exclude('exporters.tests.files') +if not have['tornado']: + sec.exclude('nbconvert.post_processors.serve') + sec.exclude('nbconvert.post_processors.tests.test_serve') - if have['qt']: - nose_pkg_names.append('qt') - - if have['tornado']: - nose_pkg_names.append('html') - - if have['zmq']: - nose_pkg_names.append('kernel') - nose_pkg_names.append('kernel.inprocess') - if inc_slow: - nose_pkg_names.append('parallel') - - if all((have['pygments'], have['jinja2'], have['sphinx'])): - nose_pkg_names.append('nbconvert') +#----------------------------------------------------------------------------- +# Functions and classes +#----------------------------------------------------------------------------- - # For debugging this code, only load quick stuff - #nose_pkg_names = ['core', 'extensions'] # dbg +def check_exclusions_exist(): + from IPython.utils.path import get_ipython_package_dir + from IPython.utils.warn import warn + parent = os.path.dirname(get_ipython_package_dir()) + for sec in test_sections: + for pattern in sec.exclusions: + fullpath = pjoin(parent, pattern) + if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'): + warn("Excluding nonexistent file: %r" % pattern) - # Make fully qualified package names prepending 'IPython.' to our name lists - nose_packages = ['IPython.%s' % m for m in nose_pkg_names ] - # Make runners - runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ] +class ExclusionPlugin(Plugin): + """A nose plugin to effect our exclusions of files and directories. + """ + name = 'exclusions' + score = 3000 # Should come before any other plugins - for name in special_test_suites: - runners.append((name, IPTester('iptest', params=name))) + def __init__(self, exclude_patterns=None): + """ + Parameters + ---------- + + exclude_patterns : sequence of strings, optional + Filenames containing these patterns (as raw strings, not as regular + expressions) are excluded from the tests. + """ + self.exclude_patterns = exclude_patterns or [] + super(ExclusionPlugin, self).__init__() + + def options(self, parser, env=os.environ): + Plugin.options(self, parser, env) + + def configure(self, options, config): + Plugin.configure(self, options, config) + # Override nose trying to disable plugin. + self.enabled = True + + def wantFile(self, filename): + """Return whether the given filename should be scanned for tests. + """ + if any(pat in filename for pat in self.exclude_patterns): + return False + return None - return runners + def wantDirectory(self, directory): + """Return whether the given directory should be scanned for tests. + """ + if any(pat in directory for pat in self.exclude_patterns): + return False + return None def run_iptest(): @@ -487,11 +367,16 @@ def run_iptest(): warnings.filterwarnings('ignore', 'This will be removed soon. Use IPython.testing.util instead') - if sys.argv[1] in special_test_suites: - sys.argv[1:2] = special_test_suites[sys.argv[1]] - special_suite = True + arg1 = sys.argv[1] + if arg1 in test_sections: + section = test_sections[arg1] + sys.argv[1:2] = section.includes + elif arg1.startswith('IPython.') and arg1[8:] in test_sections: + section = test_sections[arg1[8:]] + sys.argv[1:2] = section.includes else: - special_suite = False + section = TestSection(arg1, includes=[arg1]) + argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks @@ -522,8 +407,11 @@ def run_iptest(): # use our plugin for doctesting. It will remove the standard doctest plugin # if it finds it enabled - ipdt = IPythonDoctest() if special_suite else IPythonDoctest(make_exclude()) - plugins = [ipdt, KnownFailure()] + plugins = [ExclusionPlugin(section.excludes), IPythonDoctest(), KnownFailure()] + + # Use working directory set by parent process (see iptestcontroller) + if 'IPTEST_WORKING_DIR' in os.environ: + os.chdir(os.environ['IPTEST_WORKING_DIR']) # We need a global ipython running in this process, but the special # in-process group spawns its own IPython kernels, so for *that* group we @@ -532,97 +420,13 @@ def run_iptest(): # assumptions about what needs to be a singleton and what doesn't (app # objects should, individual shells shouldn't). But for now, this # workaround allows the test suite for the inprocess module to complete. - if not 'IPython.kernel.inprocess' in sys.argv: + if 'kernel.inprocess' not in section.name: + from IPython.testing import globalipapp globalipapp.start_ipython() # Now nose can run TestProgram(argv=argv, addplugins=plugins) - -def run_iptestall(inc_slow=False): - """Run the entire IPython test suite by calling nose and trial. - - This function constructs :class:`IPTester` instances for all IPython - modules and package and then runs each of them. This causes the modules - and packages of IPython to be tested each in their own subprocess using - nose. - - Parameters - ---------- - - inc_slow : bool, optional - Include slow tests, like IPython.parallel. By default, these tests aren't - run. - """ - - runners = make_runners(inc_slow=inc_slow) - - # Run the test runners in a temporary dir so we can nuke it when finished - # to clean up any junk files left over by accident. This also makes it - # robust against being run in non-writeable directories by mistake, as the - # temp dir will always be user-writeable. - curdir = os.getcwdu() - testdir = tempfile.gettempdir() - os.chdir(testdir) - - # Run all test runners, tracking execution time - failed = [] - t_start = time.time() - try: - for (name, runner) in runners: - print('*'*70) - print('IPython test group:',name) - res = runner.run() - if res: - failed.append( (name, runner) ) - if res == -signal.SIGINT: - print("Interrupted") - break - finally: - os.chdir(curdir) - t_end = time.time() - t_tests = t_end - t_start - nrunners = len(runners) - nfail = len(failed) - # summarize results - print() - print('*'*70) - print('Test suite completed for system with the following information:') - print(report()) - print('Ran %s test groups in %.3fs' % (nrunners, t_tests)) - print() - print('Status:') - if not failed: - print('OK') - else: - # If anything went wrong, point out what command to rerun manually to - # see the actual errors and individual summary - print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners)) - for name, failed_runner in failed: - print('-'*40) - print('Runner failed:',name) - print('You may wish to rerun this one individually, with:') - failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args] - print(u' '.join(failed_call_args)) - print() - # Ensure that our exit code indicates failure - sys.exit(1) - - -def main(): - for arg in sys.argv[1:]: - if arg.startswith('IPython') or arg in special_test_suites: - # This is in-process - run_iptest() - else: - if "--all" in sys.argv: - sys.argv.remove("--all") - inc_slow = True - else: - inc_slow = False - # This starts subprocesses - run_iptestall(inc_slow=inc_slow) - - if __name__ == '__main__': - main() + run_iptest() + diff --git a/IPython/testing/iptestcontroller.py b/IPython/testing/iptestcontroller.py new file mode 100644 index 0000000..c77d42e --- /dev/null +++ b/IPython/testing/iptestcontroller.py @@ -0,0 +1,423 @@ +# -*- coding: utf-8 -*- +"""IPython Test Process Controller + +This module runs one or more subprocesses which will actually run the IPython +test suite. + +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2009-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +from __future__ import print_function + +import argparse +import multiprocessing.pool +import os +import shutil +import signal +import sys +import subprocess +import time + +from .iptest import have, test_group_names, test_sections +from IPython.utils.py3compat import bytes_to_str +from IPython.utils.sysinfo import sys_info +from IPython.utils.tempdir import TemporaryDirectory + + +class TestController(object): + """Run tests in a subprocess + """ + #: str, IPython test suite to be executed. + section = None + #: list, command line arguments to be executed + cmd = None + #: dict, extra environment variables to set for the subprocess + env = None + #: list, TemporaryDirectory instances to clear up when the process finishes + dirs = None + #: subprocess.Popen instance + process = None + #: str, process stdout+stderr + stdout = None + #: bool, whether to capture process stdout & stderr + buffer_output = False + + def __init__(self): + self.cmd = [] + self.env = {} + self.dirs = [] + + @property + def will_run(self): + """Override in subclasses to check for dependencies.""" + return False + + def launch(self): + # print('*** ENV:', self.env) # dbg + # print('*** CMD:', self.cmd) # dbg + env = os.environ.copy() + env.update(self.env) + output = subprocess.PIPE if self.buffer_output else None + stdout = subprocess.STDOUT if self.buffer_output else None + self.process = subprocess.Popen(self.cmd, stdout=output, + stderr=stdout, env=env) + + def wait(self): + self.stdout, _ = self.process.communicate() + return self.process.returncode + + def cleanup_process(self): + """Cleanup on exit by killing any leftover processes.""" + subp = self.process + if subp is None or (subp.poll() is not None): + return # Process doesn't exist, or is already dead. + + try: + print('Cleaning up stale PID: %d' % subp.pid) + subp.kill() + except: # (OSError, WindowsError) ? + # This is just a best effort, if we fail or the process was + # really gone, ignore it. + pass + else: + for i in range(10): + if subp.poll() is None: + time.sleep(0.1) + else: + break + + if subp.poll() is None: + # The process did not die... + print('... failed. Manual cleanup may be required.') + + def cleanup(self): + "Kill process if it's still alive, and clean up temporary directories" + self.cleanup_process() + for td in self.dirs: + td.cleanup() + + __del__ = cleanup + +class PyTestController(TestController): + """Run Python tests using IPython.testing.iptest""" + #: str, Python command to execute in subprocess + pycmd = None + + def __init__(self, section): + """Create new test runner.""" + TestController.__init__(self) + self.section = section + # pycmd is put into cmd[2] in PyTestController.launch() + self.cmd = [sys.executable, '-c', None, section] + self.pycmd = "from IPython.testing.iptest import run_iptest; run_iptest()" + ipydir = TemporaryDirectory() + self.dirs.append(ipydir) + self.env['IPYTHONDIR'] = ipydir.name + self.workingdir = workingdir = TemporaryDirectory() + self.dirs.append(workingdir) + self.env['IPTEST_WORKING_DIR'] = workingdir.name + # This means we won't get odd effects from our own matplotlib config + self.env['MPLCONFIGDIR'] = workingdir.name + + @property + def will_run(self): + try: + return test_sections[self.section].will_run + except KeyError: + return True + + def add_xunit(self): + xunit_file = os.path.abspath(self.section + '.xunit.xml') + self.cmd.extend(['--with-xunit', '--xunit-file', xunit_file]) + + def add_coverage(self): + try: + sources = test_sections[self.section].includes + except KeyError: + sources = ['IPython'] + + coverage_rc = ("[run]\n" + "data_file = {data_file}\n" + "source =\n" + " {source}\n" + ).format(data_file=os.path.abspath('.coverage.'+self.section), + source="\n ".join(sources)) + config_file = os.path.join(self.workingdir.name, '.coveragerc') + with open(config_file, 'w') as f: + f.write(coverage_rc) + + self.env['COVERAGE_PROCESS_START'] = config_file + self.pycmd = "import coverage; coverage.process_startup(); " + self.pycmd + + def launch(self): + self.cmd[2] = self.pycmd + super(PyTestController, self).launch() + + +def prepare_py_test_controllers(inc_slow=False): + """Returns an ordered list of PyTestController instances to be run.""" + to_run, not_run = [], [] + if not inc_slow: + test_sections['parallel'].enabled = False + + for name in test_group_names: + controller = PyTestController(name) + if controller.will_run: + to_run.append(controller) + else: + not_run.append(controller) + return to_run, not_run + +def configure_controllers(controllers, xunit=False, coverage=False, extra_args=()): + """Apply options for a collection of TestController objects.""" + for controller in controllers: + if xunit: + controller.add_xunit() + if coverage: + controller.add_coverage() + controller.cmd.extend(extra_args) + +def do_run(controller): + try: + try: + controller.launch() + except Exception: + import traceback + traceback.print_exc() + return controller, 1 # signal failure + + exitcode = controller.wait() + return controller, exitcode + + except KeyboardInterrupt: + return controller, -signal.SIGINT + finally: + controller.cleanup() + +def report(): + """Return a string with a summary report of test-related variables.""" + + out = [ sys_info(), '\n'] + + avail = [] + not_avail = [] + + for k, is_avail in have.items(): + if is_avail: + avail.append(k) + else: + not_avail.append(k) + + if avail: + out.append('\nTools and libraries available at test time:\n') + avail.sort() + out.append(' ' + ' '.join(avail)+'\n') + + if not_avail: + out.append('\nTools and libraries NOT available at test time:\n') + not_avail.sort() + out.append(' ' + ' '.join(not_avail)+'\n') + + return ''.join(out) + +def run_iptestall(options): + """Run the entire IPython test suite by calling nose and trial. + + This function constructs :class:`IPTester` instances for all IPython + modules and package and then runs each of them. This causes the modules + and packages of IPython to be tested each in their own subprocess using + nose. + + Parameters + ---------- + + All parameters are passed as attributes of the options object. + + testgroups : list of str + Run only these sections of the test suite. If empty, run all the available + sections. + + fast : int or None + Run the test suite in parallel, using n simultaneous processes. If None + is passed, one process is used per CPU core. Default 1 (i.e. sequential) + + inc_slow : bool + Include slow tests, like IPython.parallel. By default, these tests aren't + run. + + xunit : bool + Produce Xunit XML output. This is written to multiple foo.xunit.xml files. + + coverage : bool or str + Measure code coverage from tests. True will store the raw coverage data, + or pass 'html' or 'xml' to get reports. + + extra_args : list + Extra arguments to pass to the test subprocesses, e.g. '-v' + """ + if options.fast != 1: + # If running in parallel, capture output so it doesn't get interleaved + TestController.buffer_output = True + + if options.testgroups: + to_run = [PyTestController(name) for name in options.testgroups] + not_run = [] + else: + to_run, not_run = prepare_py_test_controllers(inc_slow=options.all) + + configure_controllers(to_run, xunit=options.xunit, coverage=options.coverage, + extra_args=options.extra_args) + + def justify(ltext, rtext, width=70, fill='-'): + ltext += ' ' + rtext = (' ' + rtext).rjust(width - len(ltext), fill) + return ltext + rtext + + # Run all test runners, tracking execution time + failed = [] + t_start = time.time() + + print() + if options.fast == 1: + # This actually means sequential, i.e. with 1 job + for controller in to_run: + print('IPython test group:', controller.section) + sys.stdout.flush() # Show in correct order when output is piped + controller, res = do_run(controller) + if res: + failed.append(controller) + if res == -signal.SIGINT: + print("Interrupted") + break + print() + + else: + # Run tests concurrently + try: + pool = multiprocessing.pool.ThreadPool(options.fast) + for (controller, res) in pool.imap_unordered(do_run, to_run): + res_string = 'OK' if res == 0 else 'FAILED' + print(justify('IPython test group: ' + controller.section, res_string)) + if res: + print(bytes_to_str(controller.stdout)) + failed.append(controller) + if res == -signal.SIGINT: + print("Interrupted") + break + except KeyboardInterrupt: + return + + for controller in not_run: + print(justify('IPython test group: ' + controller.section, 'NOT RUN')) + + t_end = time.time() + t_tests = t_end - t_start + nrunners = len(to_run) + nfail = len(failed) + # summarize results + print('_'*70) + print('Test suite completed for system with the following information:') + print(report()) + print('Ran %s test groups in %.3fs' % (nrunners, t_tests)) + print() + print('Status: ', end='') + if not failed: + print('OK') + else: + # If anything went wrong, point out what command to rerun manually to + # see the actual errors and individual summary + failed_sections = [c.section for c in failed] + print('ERROR - {} out of {} test groups failed ({}).'.format(nfail, + nrunners, ', '.join(failed_sections))) + print() + print('You may wish to rerun these, with:') + print(' iptest', *failed_sections) + print() + + if options.coverage: + from coverage import coverage + cov = coverage(data_file='.coverage') + cov.combine() + cov.save() + + # Coverage HTML report + if options.coverage == 'html': + html_dir = 'ipy_htmlcov' + shutil.rmtree(html_dir, ignore_errors=True) + print("Writing HTML coverage report to %s/ ... " % html_dir, end="") + sys.stdout.flush() + + # Custom HTML reporter to clean up module names. + from coverage.html import HtmlReporter + class CustomHtmlReporter(HtmlReporter): + def find_code_units(self, morfs): + super(CustomHtmlReporter, self).find_code_units(morfs) + for cu in self.code_units: + nameparts = cu.name.split(os.sep) + if 'IPython' not in nameparts: + continue + ix = nameparts.index('IPython') + cu.name = '.'.join(nameparts[ix:]) + + # Reimplement the html_report method with our custom reporter + cov._harvest_data() + cov.config.from_args(omit='*%stests' % os.sep, html_dir=html_dir, + html_title='IPython test coverage', + ) + reporter = CustomHtmlReporter(cov, cov.config) + reporter.report(None) + print('done.') + + # Coverage XML report + elif options.coverage == 'xml': + cov.xml_report(outfile='ipy_coverage.xml') + + if failed: + # Ensure that our exit code indicates failure + sys.exit(1) + + +def main(): + # Arguments after -- should be passed through to nose. Argparse treats + # everything after -- as regular positional arguments, so we separate them + # first. + try: + ix = sys.argv.index('--') + except ValueError: + to_parse = sys.argv[1:] + extra_args = [] + else: + to_parse = sys.argv[1:ix] + extra_args = sys.argv[ix+1:] + + parser = argparse.ArgumentParser(description='Run IPython test suite') + parser.add_argument('testgroups', nargs='*', + help='Run specified groups of tests. If omitted, run ' + 'all tests.') + parser.add_argument('--all', action='store_true', + help='Include slow tests not run by default.') + parser.add_argument('-j', '--fast', nargs='?', const=None, default=1, type=int, + help='Run test sections in parallel.') + parser.add_argument('--xunit', action='store_true', + help='Produce Xunit XML results') + parser.add_argument('--coverage', nargs='?', const=True, default=False, + help="Measure test coverage. Specify 'html' or " + "'xml' to get reports.") + + options = parser.parse_args(to_parse) + options.extra_args = extra_args + + run_iptestall(options) + + +if __name__ == '__main__': + main() diff --git a/IPython/testing/ipunittest.py b/IPython/testing/ipunittest.py index 67f0f83..ae134f2 100644 --- a/IPython/testing/ipunittest.py +++ b/IPython/testing/ipunittest.py @@ -37,16 +37,9 @@ from __future__ import absolute_import # Stdlib import re -import sys import unittest from doctest import DocTestFinder, DocTestRunner, TestResults -# We already have python3-compliant code for parametric tests -if sys.version[0]=='2': - from ._paramtestpy2 import ParametricTestCase -else: - from ._paramtestpy3 import ParametricTestCase - #----------------------------------------------------------------------------- # Classes and functions #----------------------------------------------------------------------------- diff --git a/IPython/testing/plugin/ipdoctest.py b/IPython/testing/plugin/ipdoctest.py index 33a7084..3022b77 100644 --- a/IPython/testing/plugin/ipdoctest.py +++ b/IPython/testing/plugin/ipdoctest.py @@ -597,23 +597,6 @@ class ExtensionDoctest(doctests.Doctest): name = 'extdoctest' # call nosetests with --with-extdoctest enabled = True - def __init__(self,exclude_patterns=None): - """Create a new ExtensionDoctest plugin. - - Parameters - ---------- - - exclude_patterns : sequence of strings, optional - These patterns are compiled as regular expressions, subsequently used - to exclude any filename which matches them from inclusion in the test - suite (using pattern.search(), NOT pattern.match() ). - """ - - if exclude_patterns is None: - exclude_patterns = [] - self.exclude_patterns = map(re.compile,exclude_patterns) - doctests.Doctest.__init__(self) - def options(self, parser, env=os.environ): Plugin.options(self, parser, env) parser.add_option('--doctest-tests', action='store_true', @@ -716,35 +699,6 @@ class ExtensionDoctest(doctests.Doctest): else: yield False # no tests to load - def wantFile(self,filename): - """Return whether the given filename should be scanned for tests. - - Modified version that accepts extension modules as valid containers for - doctests. - """ - #print '*** ipdoctest- wantFile:',filename # dbg - - for pat in self.exclude_patterns: - if pat.search(filename): - # print '###>>> SKIP:',filename # dbg - return False - - if is_extension_module(filename): - return True - else: - return doctests.Doctest.wantFile(self,filename) - - def wantDirectory(self, directory): - """Return whether the given directory should be scanned for tests. - - Modified version that supports exclusions. - """ - - for pat in self.exclude_patterns: - if pat.search(directory): - return False - return True - class IPythonDoctest(ExtensionDoctest): """Nose Plugin that supports doctests in extension modules. diff --git a/IPython/testing/tests/test_decorators.py b/IPython/testing/tests/test_decorators.py index cb87d2f..340c00a 100644 --- a/IPython/testing/tests/test_decorators.py +++ b/IPython/testing/tests/test_decorators.py @@ -12,7 +12,6 @@ import nose.tools as nt # Our own from IPython.testing import decorators as dec from IPython.testing.skipdoctest import skip_doctest -from IPython.testing.ipunittest import ParametricTestCase #----------------------------------------------------------------------------- # Utilities @@ -47,24 +46,6 @@ def trivial(): """A trivial test""" pass -# Some examples of parametric tests. - -def is_smaller(i,j): - assert i>> d = {'a': 1, 'b': 2, 'c': 3} >>> with preserve_keys(d, 'b', 'c', 'd'): diff --git a/IPython/utils/encoding.py b/IPython/utils/encoding.py index cff1906..7eb7f2a 100644 --- a/IPython/utils/encoding.py +++ b/IPython/utils/encoding.py @@ -15,6 +15,7 @@ Utilities for dealing with text encodings #----------------------------------------------------------------------------- import sys import locale +import warnings # to deal with the possibility of sys.std* not being a stream at all def get_stream_enc(stream, default=None): @@ -51,6 +52,16 @@ def getdefaultencoding(): enc = locale.getpreferredencoding() except Exception: pass - return enc or sys.getdefaultencoding() + enc = enc or sys.getdefaultencoding() + # On windows `cp0` can be returned to indicate that there is no code page. + # Since cp0 is an invalid encoding return instead cp1252 which is the + # Western European default. + if enc == 'cp0': + warnings.warn( + "Invalid code page cp0 detected - using cp1252 instead." + "If cp1252 is incorrect please ensure a valid code page " + "is defined for the process.", RuntimeWarning) + return 'cp1252' + return enc DEFAULT_ENCODING = getdefaultencoding() diff --git a/IPython/utils/localinterfaces.py b/IPython/utils/localinterfaces.py index 334e0f4..e67a35f 100644 --- a/IPython/utils/localinterfaces.py +++ b/IPython/utils/localinterfaces.py @@ -29,27 +29,80 @@ from .data import uniq_stable #----------------------------------------------------------------------------- LOCAL_IPS = [] -try: - LOCAL_IPS = socket.gethostbyname_ex('localhost')[2] -except socket.error: - pass - PUBLIC_IPS = [] -try: - hostname = socket.gethostname() - PUBLIC_IPS = socket.gethostbyname_ex(hostname)[2] - # try hostname.local, in case hostname has been short-circuited to loopback - if not hostname.endswith('.local') and all(ip.startswith('127') for ip in PUBLIC_IPS): - PUBLIC_IPS = socket.gethostbyname_ex(socket.gethostname() + '.local')[2] -except socket.error: - pass -else: - PUBLIC_IPS = uniq_stable(PUBLIC_IPS) - LOCAL_IPS.extend(PUBLIC_IPS) - -# include all-interface aliases: 0.0.0.0 and '' -LOCAL_IPS.extend(['0.0.0.0', '']) - -LOCAL_IPS = uniq_stable(LOCAL_IPS) - -LOCALHOST = LOCAL_IPS[0] + +LOCALHOST = '127.0.0.1' + +def _only_once(f): + """decorator to only run a function once""" + f.called = False + def wrapped(): + if f.called: + return + ret = f() + f.called = True + return ret + return wrapped + +def _requires_ips(f): + """decorator to ensure load_ips has been run before f""" + def ips_loaded(*args, **kwargs): + _load_ips() + return f(*args, **kwargs) + return ips_loaded + +@_only_once +def _load_ips(): + """load the IPs that point to this machine + + This function will only ever be called once. + """ + global LOCALHOST + try: + LOCAL_IPS[:] = socket.gethostbyname_ex('localhost')[2] + except socket.error: + pass + + try: + hostname = socket.gethostname() + PUBLIC_IPS[:] = socket.gethostbyname_ex(hostname)[2] + # try hostname.local, in case hostname has been short-circuited to loopback + if not hostname.endswith('.local') and all(ip.startswith('127') for ip in PUBLIC_IPS): + PUBLIC_IPS[:] = socket.gethostbyname_ex(socket.gethostname() + '.local')[2] + except socket.error: + pass + finally: + PUBLIC_IPS[:] = uniq_stable(PUBLIC_IPS) + LOCAL_IPS.extend(PUBLIC_IPS) + + # include all-interface aliases: 0.0.0.0 and '' + LOCAL_IPS.extend(['0.0.0.0', '']) + + LOCAL_IPS[:] = uniq_stable(LOCAL_IPS) + + LOCALHOST = LOCAL_IPS[0] + +@_requires_ips +def local_ips(): + """return the IP addresses that point to this machine""" + return LOCAL_IPS + +@_requires_ips +def public_ips(): + """return the IP addresses for this machine that are visible to other machines""" + return PUBLIC_IPS + +@_requires_ips +def localhost(): + """return ip for localhost (almost always 127.0.0.1)""" + return LOCALHOST + +@_requires_ips +def is_local_ip(ip): + """does `ip` point to this machine?""" + return ip in LOCAL_IPS + +@_requires_ips +def is_public_ip(ip): + """is `ip` a publicly visible address?""" + return ip in PUBLIC_IPS diff --git a/IPython/utils/pickleutil.py b/IPython/utils/pickleutil.py index 04ac882..61315c1 100644 --- a/IPython/utils/pickleutil.py +++ b/IPython/utils/pickleutil.py @@ -25,11 +25,6 @@ try: except ImportError: import pickle -try: - import numpy -except: - numpy = None - import codeutil # This registers a hook when it's imported import py3compat from importstring import import_item @@ -163,6 +158,7 @@ class CannedClass(CannedObject): class CannedArray(CannedObject): def __init__(self, obj): + from numpy import ascontiguousarray self.shape = obj.shape self.dtype = obj.dtype.descr if obj.dtype.fields else obj.dtype.str if sum(obj.shape) == 0: @@ -170,16 +166,17 @@ class CannedArray(CannedObject): self.buffers = [pickle.dumps(obj, -1)] else: # ensure contiguous - obj = numpy.ascontiguousarray(obj, dtype=None) + obj = ascontiguousarray(obj, dtype=None) self.buffers = [buffer(obj)] def get_object(self, g=None): + from numpy import frombuffer data = self.buffers[0] if sum(self.shape) == 0: # no shape, we just pickled it return pickle.loads(data) else: - return numpy.frombuffer(data, dtype=self.dtype).reshape(self.shape) + return frombuffer(data, dtype=self.dtype).reshape(self.shape) class CannedBytes(CannedObject): @@ -225,7 +222,7 @@ def _import_mapping(mapping, original=None): except Exception: if original and key not in original: # only message on user-added classes - log.error("cannning class not importable: %r", key, exc_info=True) + log.error("canning class not importable: %r", key, exc_info=True) mapping.pop(key) else: mapping[cls] = mapping.pop(key) diff --git a/IPython/utils/py3compat.py b/IPython/utils/py3compat.py index 9731368..f52a80d 100644 --- a/IPython/utils/py3compat.py +++ b/IPython/utils/py3compat.py @@ -60,12 +60,12 @@ def safe_unicode(e): pass try: - return py3compat.str_to_unicode(str(e)) + return str_to_unicode(str(e)) except UnicodeError: pass try: - return py3compat.str_to_unicode(repr(e)) + return str_to_unicode(repr(e)) except UnicodeError: pass diff --git a/IPython/utils/sysinfo.py b/IPython/utils/sysinfo.py index 98723df..69abde2 100644 --- a/IPython/utils/sysinfo.py +++ b/IPython/utils/sysinfo.py @@ -98,18 +98,20 @@ def pkg_info(pkg_path): def sys_info(): """Return useful information about IPython and the system, as a string. - Example - ------- - In [2]: print sys_info() - {'commit_hash': '144fdae', # random - 'commit_source': 'repository', - 'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython', - 'ipython_version': '0.11.dev', - 'os_name': 'posix', - 'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick', - 'sys_executable': '/usr/bin/python', - 'sys_platform': 'linux2', - 'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'} + Examples + -------- + :: + + In [2]: print sys_info() + {'commit_hash': '144fdae', # random + 'commit_source': 'repository', + 'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython', + 'ipython_version': '0.11.dev', + 'os_name': 'posix', + 'platform': 'Linux-2.6.35-22-generic-i686-with-Ubuntu-10.10-maverick', + 'sys_executable': '/usr/bin/python', + 'sys_platform': 'linux2', + 'sys_version': '2.6.6 (r266:84292, Sep 15 2010, 15:52:39) \\n[GCC 4.4.5]'} """ p = os.path path = p.dirname(p.abspath(p.join(__file__, '..'))) diff --git a/IPython/utils/tempdir.py b/IPython/utils/tempdir.py index 0aa7440..0f27889 100644 --- a/IPython/utils/tempdir.py +++ b/IPython/utils/tempdir.py @@ -5,6 +5,8 @@ This is copied from the stdlib and will be standard in Python 3.2 and onwards. from __future__ import print_function import os as _os +import warnings as _warnings +import sys as _sys # This code should only be used in Python versions < 3.2, since after that we # can rely on the stdlib itself. @@ -49,7 +51,7 @@ except ImportError: self._closed = True if _warn: self._warn("Implicitly cleaning up {!r}".format(self), - ResourceWarning) + Warning) def __exit__(self, exc, value, tb): self.cleanup() @@ -69,6 +71,7 @@ except ImportError: _remove = staticmethod(_os.remove) _rmdir = staticmethod(_os.rmdir) _os_error = _os.error + _warn = _warnings.warn def _rmtree(self, path): # Essentially a stripped down version of shutil.rmtree. We can't diff --git a/IPython/utils/tests/test_io.py b/IPython/utils/tests/test_io.py index 221fdb4..2ece068 100644 --- a/IPython/utils/tests/test_io.py +++ b/IPython/utils/tests/test_io.py @@ -17,10 +17,10 @@ import sys from StringIO import StringIO from subprocess import Popen, PIPE +import unittest import nose.tools as nt -from IPython.testing.ipunittest import ParametricTestCase from IPython.utils.io import Tee, capture_output from IPython.utils.py3compat import doctest_refactor_print @@ -38,7 +38,7 @@ def test_tee_simple(): nt.assert_equal(chan.getvalue(), text+"\n") -class TeeTestCase(ParametricTestCase): +class TeeTestCase(unittest.TestCase): def tchan(self, channel, check='close'): trap = StringIO() @@ -61,7 +61,7 @@ class TeeTestCase(ParametricTestCase): def test(self): for chan in ['stdout', 'stderr']: for check in ['close', 'del']: - yield self.tchan(chan, check) + self.tchan(chan, check) def test_io_init(): """Test that io.stdin/out/err exist at startup""" diff --git a/IPython/utils/tests/test_jsonutil.py b/IPython/utils/tests/test_jsonutil.py index 61f1df7..847f54b 100644 --- a/IPython/utils/tests/test_jsonutil.py +++ b/IPython/utils/tests/test_jsonutil.py @@ -19,7 +19,6 @@ from base64 import decodestring import nose.tools as nt # our own -from IPython.testing import decorators as dec from IPython.utils import jsonutil, tz from ..jsonutil import json_clean, encode_images from ..py3compat import unicode_to_str, str_to_bytes @@ -62,7 +61,6 @@ def test(): -@dec.parametric def test_encode_images(): # invalid data, but the header and footer are from real files pngdata = b'\x89PNG\r\n\x1a\nblahblahnotactuallyvalidIEND\xaeB`\x82' @@ -76,19 +74,19 @@ def test_encode_images(): for key, value in fmt.iteritems(): # encoded has unicode, want bytes decoded = decodestring(encoded[key].encode('ascii')) - yield nt.assert_equal(decoded, value) + nt.assert_equal(decoded, value) encoded2 = encode_images(encoded) - yield nt.assert_equal(encoded, encoded2) + nt.assert_equal(encoded, encoded2) b64_str = {} for key, encoded in encoded.iteritems(): b64_str[key] = unicode_to_str(encoded) encoded3 = encode_images(b64_str) - yield nt.assert_equal(encoded3, b64_str) + nt.assert_equal(encoded3, b64_str) for key, value in fmt.iteritems(): # encoded3 has str, want bytes decoded = decodestring(str_to_bytes(encoded3[key])) - yield nt.assert_equal(decoded, value) + nt.assert_equal(decoded, value) def test_lambda(): jc = json_clean(lambda : 1) diff --git a/IPython/utils/text.py b/IPython/utils/text.py index 55b8861..2980e0e 100644 --- a/IPython/utils/text.py +++ b/IPython/utils/text.py @@ -21,6 +21,7 @@ Inheritance diagram: import os import re +import sys import textwrap from string import Formatter @@ -28,6 +29,18 @@ from IPython.external.path import path from IPython.testing.skipdoctest import skip_doctest_py3, skip_doctest from IPython.utils import py3compat + +#----------------------------------------------------------------------------- +# Declarations +#----------------------------------------------------------------------------- + +# datetime.strftime date format for ipython +if sys.platform == 'win32': + date_format = "%B %d, %Y" +else: + date_format = "%B %-d, %Y" + + #----------------------------------------------------------------------------- # Code #----------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index fabc39e..9ac3076 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,8 @@ For full details, see the installation section of the manual. The basic parts of IPython only need the Python standard library, but much of its more advanced functionality requires extra packages. -Officially, IPython requires Python version 2.6, 2.7, or 3.1 and above. +Officially, IPython requires Python version 2.7, or 3.3 and above. +IPython 1.x is the last IPython version to support Python 2.6 and 3.2. Instant running @@ -38,7 +39,9 @@ If you want to hack on certain parts, e.g. the IPython notebook, in a clean environment (such as a virtualenv) you can use ``pip`` to grab the necessary dependencies quickly:: - $ pip install -e .[notebook] + $ git clone --recursive https://github.com/ipython/ipython.git + $ cd ipython + $ pip install -e ".[notebook]" This installs the necessary packages and symlinks IPython into your current environment so that you can work on your local repo copy and run it from anywhere:: diff --git a/docs/Makefile b/docs/Makefile index c5d2df4..49886b1 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -52,7 +52,10 @@ dist: html cp -al build/html . @echo "Build finished. Final docs are in html/" -html: api +html: api +html_noapi: clean + +html html_noapi: mkdir -p build/html build/doctrees $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) build/html @echo diff --git a/docs/autogen_api.py b/docs/autogen_api.py index 74f841d..b0d831e 100755 --- a/docs/autogen_api.py +++ b/docs/autogen_api.py @@ -35,7 +35,7 @@ if __name__ == '__main__': r'\.zmq', ] - docwriter.module_skip_patterns += [ r'\.core\.fakemodule', + docwriter.module_skip_patterns += [ r'\.testing\.iptest', # Keeping these disabled is OK r'\.parallel\.controller\.mongodb', diff --git a/docs/source/conf.py b/docs/source/conf.py index 0764a40..6281a94 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -71,7 +71,7 @@ if iprelease['_version_extra']: .. note:: This documentation is for a development version of IPython. There may be - significant differences from the latest stable release (0.13.2). + significant differences from the latest stable release (1.1.0). """ diff --git a/docs/source/development/messaging.rst b/docs/source/development/messaging.rst index 8919b44..5fcd571 100644 --- a/docs/source/development/messaging.rst +++ b/docs/source/development/messaging.rst @@ -984,6 +984,20 @@ Message type: ``status``:: execution_state : ('busy', 'idle', 'starting') } +Clear output +------------ + +This message type is used to clear the output that is visible on the frontend. + +Message type: ``clear_output``:: + + content = { + + # Wait to clear the output until new output is available. Clears the + # existing output immediately before the new output is displayed. + # Useful for creating simple animations with minimal flickering. + 'wait' : bool, + } Messages on the stdin ROUTER/DEALER sockets =========================================== diff --git a/docs/source/interactive/nbconvert.rst b/docs/source/interactive/nbconvert.rst index fc437a1..1789a62 100644 --- a/docs/source/interactive/nbconvert.rst +++ b/docs/source/interactive/nbconvert.rst @@ -61,14 +61,9 @@ The currently supported export formats are: * ``--to slides`` This generates a Reveal.js HTML slideshow. - It must be served by an HTTP server. The easiest way to get this is to add + It must be served by an HTTP server. The easiest way to do this is adding ``--post serve`` on the command-line. - If you want to use the speaker notes plugin, just add - ``--slide-notes=True`` on the command-line. - For low connectivity environments, you can use a local copy of the reveal.js library, - just add ``--offline-slides=reveal.js`` on the command-line, and do not forget to move - your downloaded ``reveal.js`` library to the same folder where your slides are located. - + * ``--to markdown`` Simple markdown output. Markdown cells are unaffected, @@ -125,6 +120,21 @@ and using the command:: .. _notebook_format: +LaTeX citations +--------------- + +``nbconvert`` now has support for LaTeX citations. With this capability you +can: + +* Manage citations using BibTeX. +* Cite those citations in Markdown cells using HTML data attributes. +* Have ``nbconvert`` generate proper LaTeX citations and run BibTeX. + +For an example of how this works, please see the citations example in +the nbconvert-examples_ repository. + +.. _nbconvert-examples: https://github.com/ipython/nbconvert-examples + Notebook JSON file format ------------------------- diff --git a/docs/source/interactive/notebook.rst b/docs/source/interactive/notebook.rst index 2a727b5..7867730 100644 --- a/docs/source/interactive/notebook.rst +++ b/docs/source/interactive/notebook.rst @@ -202,6 +202,9 @@ being a **code cell**, but its type can be changed by using a dropdown on the toolbar (which will be "Code", initially), or via :ref:`keyboard shortcuts `. +For more information on the different things you can do in a notebook, +see the `collection of examples +`_. Code cells ~~~~~~~~~~ diff --git a/docs/source/interactive/public_server.rst b/docs/source/interactive/public_server.rst index 2f692fd..7e8ff97 100644 --- a/docs/source/interactive/public_server.rst +++ b/docs/source/interactive/public_server.rst @@ -60,8 +60,7 @@ the command:: the following command will create a certificate valid for 365 days with both the key and certificate data written to the same file:: - $ openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert. - pem -out mycert.pem + $ openssl req -x509 -nodes -days 365 -newkey rsa:1024 -keyout mycert.pem -out mycert.pem Your browser will warn you of a dangerous certificate because it is self-signed. If you want to have a fully compliant certificate that will not diff --git a/docs/source/whatsnew/github-stats-1.0.rst b/docs/source/whatsnew/github-stats-1.0.rst index 8fcca2b..f3beb3c 100644 --- a/docs/source/whatsnew/github-stats-1.0.rst +++ b/docs/source/whatsnew/github-stats-1.0.rst @@ -3,6 +3,127 @@ Issues closed in the 1.0 development cycle ========================================== +Issues closed in 1.1 +-------------------- + +GitHub stats for 2013/08/08 - 2013/09/09 (since 1.0) + +These lists are automatically generated, and may be incomplete or contain duplicates. + +The following 25 authors contributed 337 commits. + +* Benjamin Ragan-Kelley +* Bing Xia +* Bradley M. Froehle +* Brian E. Granger +* Damián Avila +* dhirschfeld +* Dražen Lučanin +* gmbecker +* Jake Vanderplas +* Jason Grout +* Jonathan Frederic +* Kevin Burke +* Kyle Kelley +* Matt Henderson +* Matthew Brett +* Matthias Bussonnier +* Pankaj Pandey +* Paul Ivanov +* rossant +* Samuel Ainsworth +* Stephan Rave +* stonebig +* Thomas Kluyver +* Yaroslav Halchenko +* Zachary Sailer + + +We closed a total of 76 issues, 58 pull requests and 18 regular issues; +this is the full list (generated with the script :file:`tools/github_stats.py`): + +Pull Requests (58): + +* :ghpull:`4188`: Allow user_ns trait to be None +* :ghpull:`4189`: always fire LOCAL_IPS.extend(PUBLIC_IPS) +* :ghpull:`4174`: various issues in markdown and rst templates +* :ghpull:`4178`: add missing data_javascript +* :ghpull:`4181`: nbconvert: Fix, sphinx template not removing new lines from headers +* :ghpull:`4043`: don't 'restore_bytes' in from_JSON +* :ghpull:`4163`: Fix for incorrect default encoding on Windows. +* :ghpull:`4136`: catch javascript errors in any output +* :ghpull:`4171`: add nbconvert config file when creating profiles +* :ghpull:`4125`: Basic exercise of `ipython [subcommand] -h` and help-all +* :ghpull:`4085`: nbconvert: Fix sphinx preprocessor date format string for Windows +* :ghpull:`4159`: don't split `.cell` and `div.cell` CSS +* :ghpull:`4158`: generate choices for `--gui` configurable from real mapping +* :ghpull:`4065`: do not include specific css in embedable one +* :ghpull:`4092`: nbconvert: Fix for unicode html headers, Windows + Python 2.x +* :ghpull:`4074`: close Client sockets if connection fails +* :ghpull:`4064`: Store default codemirror mode in only 1 place +* :ghpull:`4104`: Add way to install MathJax to a particular profile +* :ghpull:`4144`: help_end transformer shouldn't pick up ? in multiline string +* :ghpull:`4143`: update example custom.js +* :ghpull:`4142`: DOC: unwrap openssl line in public_server doc +* :ghpull:`4141`: add files with a separate `add` call in backport_pr +* :ghpull:`4137`: Restore autorestore option for storemagic +* :ghpull:`4098`: pass profile-dir instead of profile name to Kernel +* :ghpull:`4120`: support `input` in Python 2 kernels +* :ghpull:`4088`: nbconvert: Fix coalescestreams line with incorrect nesting causing strange behavior +* :ghpull:`4060`: only strip continuation prompts if regular prompts seen first +* :ghpull:`4132`: Fixed name error bug in function safe_unicode in module py3compat. +* :ghpull:`4121`: move test_kernel from IPython.zmq to IPython.kernel +* :ghpull:`4118`: ZMQ heartbeat channel: catch EINTR exceptions and continue. +* :ghpull:`4054`: use unicode for HTML export +* :ghpull:`4106`: fix a couple of default block values +* :ghpull:`4115`: Update docs on declaring a magic function +* :ghpull:`4101`: restore accidentally removed EngineError +* :ghpull:`4096`: minor docs changes +* :ghpull:`4056`: respect `pylab_import_all` when `--pylab` specified at the command-line +* :ghpull:`4091`: Make Qt console banner configurable +* :ghpull:`4086`: fix missing errno import +* :ghpull:`4030`: exclude `.git` in MANIFEST.in +* :ghpull:`4047`: Use istype() when checking if canned object is a dict +* :ghpull:`4031`: don't close_fds on Windows +* :ghpull:`4029`: bson.Binary moved +* :ghpull:`4035`: Fixed custom jinja2 templates being ignored when setting template_path +* :ghpull:`4026`: small doc fix in nbconvert +* :ghpull:`4016`: Fix IPython.start_* functions +* :ghpull:`4021`: Fix parallel.client.View map() on numpy arrays +* :ghpull:`4022`: DOC: fix links to matplotlib, notebook docs +* :ghpull:`4018`: Fix warning when running IPython.kernel tests +* :ghpull:`4019`: Test skipping without unicode paths +* :ghpull:`4008`: Transform code before %prun/%%prun runs +* :ghpull:`4014`: Fix typo in ipapp +* :ghpull:`3987`: get files list in backport_pr +* :ghpull:`3974`: nbconvert: Fix app tests on Window7 w/ Python 3.3 +* :ghpull:`3978`: fix `--existing` with non-localhost IP +* :ghpull:`3939`: minor checkpoint cleanup +* :ghpull:`3981`: BF: fix nbconvert rst input prompt spacing +* :ghpull:`3960`: Don't make sphinx a dependency for importing nbconvert +* :ghpull:`3973`: logging.Formatter is not new-style in 2.6 + +Issues (18): + +* :ghissue:`4024`: nbconvert markdown issues +* :ghissue:`4095`: Catch js error in append html in stream/pyerr +* :ghissue:`4156`: Specifying --gui=tk at the command line +* :ghissue:`3818`: nbconvert can't handle Heading with Chinese characters on Japanese Windows OS. +* :ghissue:`4134`: multi-line parser fails on ''' in comment, qtconsole and notebook. +* :ghissue:`3998`: sample custom.js needs to be updated +* :ghissue:`4078`: StoreMagic.autorestore not working in 1.0.0 +* :ghissue:`3990`: Buitlin `input` doesn't work over zmq +* :ghissue:`4015`: nbconvert fails to convert all the content of a notebook +* :ghissue:`4059`: Issues with Ellipsis literal in Python 3 +* :ghissue:`4103`: Wrong default argument of DirectView.clear +* :ghissue:`4100`: parallel.client.client references undefined error.EngineError +* :ghissue:`4005`: IPython.start_kernel doesn't work. +* :ghissue:`4020`: IPython parallel map fails on numpy arrays +* :ghissue:`3945`: nbconvert: commandline tests fail Win7x64 Py3.3 +* :ghissue:`3977`: unable to complete remote connections for two-process +* :ghissue:`3980`: nbconvert rst output lacks needed blank lines +* :ghissue:`3968`: TypeError: super() argument 1 must be type, not classobj (Python 2.6.6) + Issues closed in 1.0 -------------------- diff --git a/docs/source/whatsnew/pr/clear_output.rst b/docs/source/whatsnew/pr/clear_output.rst new file mode 100644 index 0000000..0040ad1 --- /dev/null +++ b/docs/source/whatsnew/pr/clear_output.rst @@ -0,0 +1,9 @@ +clear_output changes +-------------------- + +* There is no longer a 500ms delay when calling ``clear_output``. +* The ability to clear stderr and stdout individually was removed. +* A new ``wait`` flag that prevents ``clear_output`` from being executed until new + output is available. This eliminates animation flickering by allowing the + user to double buffer the output. +* The output div height is remembered when the ``wait=True`` flag is used. diff --git a/docs/source/whatsnew/pr/incompat-drop-alias.rst b/docs/source/whatsnew/pr/incompat-drop-alias.rst new file mode 100644 index 0000000..570a7cf --- /dev/null +++ b/docs/source/whatsnew/pr/incompat-drop-alias.rst @@ -0,0 +1,3 @@ +- The alias system has been reimplemented to use magic functions. There should be little + visible difference while automagics are enabled, as they are by default, but parts of the + :class:`~IPython.core.alias.AliasManager` API have been removed. diff --git a/docs/source/whatsnew/pr/incompat-drop-fakemodule.rst b/docs/source/whatsnew/pr/incompat-drop-fakemodule.rst new file mode 100644 index 0000000..65bfa11 --- /dev/null +++ b/docs/source/whatsnew/pr/incompat-drop-fakemodule.rst @@ -0,0 +1 @@ +* The module ``IPython.core.fakemodule`` has been removed. diff --git a/docs/source/whatsnew/pr/incompat-mpl-backend.rst b/docs/source/whatsnew/pr/incompat-mpl-backend.rst new file mode 100644 index 0000000..f62e70c --- /dev/null +++ b/docs/source/whatsnew/pr/incompat-mpl-backend.rst @@ -0,0 +1,4 @@ +We fixed an issue with switching between matplotlib inline and GUI backends, +but the fix requires matplotlib 1.1 or newer. So from now on, we consider +matplotlib 1.1 to be the minimally supported version for IPython. Older +versions for the most part will work, but we make no guarantees about it. diff --git a/docs/source/whatsnew/pr/select-notebook-rename.rst b/docs/source/whatsnew/pr/select-notebook-rename.rst new file mode 100644 index 0000000..b731736 --- /dev/null +++ b/docs/source/whatsnew/pr/select-notebook-rename.rst @@ -0,0 +1,6 @@ +Select Notebook Name When Renaming a Notebook +--------------------------------------------- + +The default notebook name is Untitled. It's unlikely you want to keep this name +or part of it when naming your notebook. Instead, IPython will select the text +in the input field so the user can easily type over the name and change it. diff --git a/examples/notebooks/Animations Using clear_output.ipynb b/examples/notebooks/Animations Using clear_output.ipynb index f3de1c9..c799dfb 100644 --- a/examples/notebooks/Animations Using clear_output.ipynb +++ b/examples/notebooks/Animations Using clear_output.ipynb @@ -21,7 +21,7 @@ "source": [ "Sometimes you want to clear the output area in the middle of a calculation. This can be useful for doing simple animations. In terminals, there is the carriage-return (`'\\r'`) for overwriting a single line, but the notebook frontend can clear the whole output area, not just a single line.\n", "\n", - "To clear output in the Notebook you can use the `clear_output` function." + "To clear output in the Notebook you can use the `clear_output()` function. If you are clearing the output every frame of an animation, calling `clear_output()` will create noticeable flickering. You can use `clear_output(wait=True)` to add the *clear_output* call to a queue. When data becomes available to replace the existing output, the *clear_output* will be called immediately before the new data is added. This avoids the flickering by not rendering the cleared output to the screen." ] }, { @@ -58,7 +58,7 @@ "from IPython.display import display, clear_output\n", "for i in range(10):\n", " time.sleep(0.25)\n", - " clear_output()\n", + " clear_output(wait=True)\n", " print(i)\n", " sys.stdout.flush()" ], @@ -166,7 +166,7 @@ "for n in range(1,10):\n", " time.sleep(1)\n", " ax.plot(x, jn(x,n))\n", - " clear_output()\n", + " clear_output(wait=True)\n", " display(f)\n", "\n", "# close the figure at the end, so we don't get a duplicate\n", diff --git a/examples/notebooks/Custom Display Logic.ipynb b/examples/notebooks/Custom Display Logic.ipynb index ea81c52..2614ba6 100644 --- a/examples/notebooks/Custom Display Logic.ipynb +++ b/examples/notebooks/Custom Display Logic.ipynb @@ -78,12 +78,12 @@ "source": [ "The main idea of the first approach is that you have to implement special display methods, one for each representation you want to use. The names of the special methods are self explanatory:\n", "\n", - "* _repr_html_\n", - "* _repr_json_\n", - "* _repr_jpeg_\n", - "* _repr_png_\n", - "* _repr_svg_\n", - "* _repr_latex_\n", + "* `_repr_html_`\n", + "* `_repr_json_`\n", + "* `_repr_jpeg_`\n", + "* `_repr_png_`\n", + "* `_repr_svg_`\n", + "* `_repr_latex_`\n", "\n", "As an illustration, we build a class that holds data generated by sampling a Gaussian distribution with given mean and variance. Each frontend can then decide which representation it will display be default. Further, we show how to display a particular representation." ] diff --git a/examples/notebooks/Frontend-Kernel Model.ipynb b/examples/notebooks/Frontend-Kernel Model.ipynb index e176151..249788b 100644 --- a/examples/notebooks/Frontend-Kernel Model.ipynb +++ b/examples/notebooks/Frontend-Kernel Model.ipynb @@ -1,6 +1,6 @@ { "metadata": { - "name": "Frontend-Kernel Model" + "name": "" }, "nbformat": 3, "nbformat_minor": 0, @@ -152,6 +152,21 @@ "cell_type": "code", "collapsed": false, "input": [ + "# Python 3 compat\n", + "try:\n", + " raw_input\n", + "except NameError:\n", + " raw_input = input" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ "name = raw_input(\"What is your name? \")\n", "name" ], @@ -169,13 +184,49 @@ { "metadata": {}, "output_type": "pyout", - "prompt_number": 1, + "prompt_number": 2, "text": [ - "u'Sir Robin'" + "'Sir Robin'" ] } ], - "prompt_number": 1 + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Python 2-only**: the eval input works as well (`input` is just `eval(raw_input(prompt))`)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "fingers = input(\"How many fingers? \")\n", + "fingers, type(fingers)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "stream": "stdout", + "text": [ + "How many fingers? 4\n" + ] + }, + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 3, + "text": [ + "(4, int)" + ] + } + ], + "prompt_number": 3 }, { "cell_type": "code", @@ -195,13 +246,13 @@ "output_type": "pyerr", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mZeroDivisionError\u001b[0m Traceback (most recent call last)", - "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[1;32m\u001b[0m in \u001b[0;36mdiv\u001b[1;34m(x, y)\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[0;32m 2\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[1;32m\u001b[0m in \u001b[0;36mdiv\u001b[1;34m(x, y)\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;31mZeroDivisionError\u001b[0m: integer division or modulo by zero" ] } ], - "prompt_number": 2 + "prompt_number": 4 }, { "cell_type": "code", @@ -216,7 +267,7 @@ "output_type": "stream", "stream": "stdout", "text": [ - "> \u001b[1;32m\u001b[0m(2)\u001b[0;36mdiv\u001b[1;34m()\u001b[0m\n", + "> \u001b[1;32m\u001b[0m(2)\u001b[0;36mdiv\u001b[1;34m()\u001b[0m\n", "\u001b[1;32m 1 \u001b[1;33m\u001b[1;32mdef\u001b[0m \u001b[0mdiv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[0m\u001b[1;32m----> 2 \u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[0m\u001b[1;32m 3 \u001b[1;33m\u001b[1;33m\u001b[0m\u001b[0m\n", @@ -262,7 +313,7 @@ ] } ], - "prompt_number": 3 + "prompt_number": 5 } ], "metadata": {} diff --git a/examples/notebooks/README.md b/examples/notebooks/README.md index 5544020..6e0fa78 100644 --- a/examples/notebooks/README.md +++ b/examples/notebooks/README.md @@ -23,7 +23,7 @@ programming to advanced topics in scientific computing. * [Octave Magic](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Octave%20Magic.ipynb) * [Part 1 - Running Code](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Part%201%20-%20Running%20Code.ipynb) * [Part 2 - Basic Output](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Part%202%20-%20Basic%20Output.ipynb) -* [Part 3 - Pylab and Matplotlib](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Part%203%20-%20Pylab%20and%20Matplotlib.ipynb) +* [Part 3 - Matplotlib](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Part%203%20-%20Plotting%20with%20Matplotlib.ipynb) * [Part 4 - Markdown Cells](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Part%204%20-%20Markdown%20Cells.ipynb) * [Part 5 - Rich Display System](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Part%205%20-%20Rich%20Display%20System.ipynb) * [Progress Bars](http://nbviewer.ipython.org/url/github.com/ipython/ipython/raw/master/examples/notebooks/Progress%20Bars.ipynb) diff --git a/examples/parallel/InteractiveMPI-publish-data.ipynb b/examples/parallel/InteractiveMPI-publish-data.ipynb index 4235c83..6205db3 100644 --- a/examples/parallel/InteractiveMPI-publish-data.ipynb +++ b/examples/parallel/InteractiveMPI-publish-data.ipynb @@ -271,7 +271,7 @@ " # We clear the notebook output before plotting this if in-place \n", " # plot updating is requested\n", " if in_place:\n", - " clear_output()\n", + " clear_output(wait=True)\n", " display(fig)\n", " \n", " return fig" @@ -333,7 +333,7 @@ " msg = 'Simulation completed!'\n", " tmon = dt.datetime.now() - t0\n", " if plots_in_place and fig is not None:\n", - " clear_output()\n", + " clear_output(wait=True)\n", " plt.close('all')\n", " display(fig)\n", " print msg\n", diff --git a/examples/parallel/InteractiveMPI.ipynb b/examples/parallel/InteractiveMPI.ipynb index ea03906..526fe29 100644 --- a/examples/parallel/InteractiveMPI.ipynb +++ b/examples/parallel/InteractiveMPI.ipynb @@ -281,7 +281,7 @@ " plt.axis('off')\n", " # We clear the notebook output before plotting this if in-place plot updating is requested\n", " if in_place:\n", - " clear_output()\n", + " clear_output(wait=True)\n", " display(fig)\n", " return fig" ], @@ -377,7 +377,7 @@ " msg = 'Simulation completed!'\n", " tmon = dt.datetime.now() - t0\n", " if plots_in_place and fig is not None:\n", - " clear_output()\n", + " clear_output(wait=True)\n", " plt.close('all')\n", " display(fig)\n", " print msg\n", diff --git a/examples/widgets/directview/directview.js b/examples/widgets/directview/directview.js index 7eed2ad..37c6ab4 100644 --- a/examples/widgets/directview/directview.js +++ b/examples/widgets/directview/directview.js @@ -108,7 +108,7 @@ DirectViewWidget.prototype.set_kernel = function (kernel) { DirectViewWidget.prototype.execute = function () { - this.output_area.clear_output(true, true, true); + this.output_area.clear_output(); this.element.addClass("running"); var callbacks = { 'execute_reply': $.proxy(this._handle_execute_reply, this), diff --git a/setup.py b/setup.py index 48e2b00..11a52d8 100755 --- a/setup.py +++ b/setup.py @@ -26,12 +26,10 @@ import sys # This check is also made in IPython/__init__, don't forget to update both when # changing Python version requirements. -#~ if sys.version[0:3] < '2.6': - #~ error = """\ -#~ ERROR: 'IPython requires Python Version 2.6 or above.' -#~ Exiting.""" - #~ print >> sys.stderr, error - #~ sys.exit(1) +if sys.version_info[:2] < (2,7): + error = "ERROR: IPython requires Python Version 2.7 or above." + print(error, file=sys.stderr) + sys.exit(1) PY3 = (sys.version_info[0] >= 3) @@ -72,6 +70,7 @@ from setupbase import ( update_submodules, require_submodules, UpdateSubmodules, + CompileCSS, ) from setupext import setupext @@ -238,6 +237,7 @@ setup_args['cmdclass'] = { 'sdist' : git_prebuild('IPython', sdist), 'upload_wininst' : UploadWindowsInstallers, 'submodule' : UpdateSubmodules, + 'css' : CompileCSS, } #--------------------------------------------------------------------------- diff --git a/setupbase.py b/setupbase.py index 81f9feb..ca87fc0 100644 --- a/setupbase.py +++ b/setupbase.py @@ -30,6 +30,7 @@ except: from distutils.command.build_py import build_py from distutils.cmd import Command from glob import glob +from subprocess import call from setupext import install_data_ext @@ -323,7 +324,7 @@ def find_scripts(entry_points=False, suffix=''): 'ipengine%s = IPython.parallel.apps.ipengineapp:launch_new_instance', 'iplogger%s = IPython.parallel.apps.iploggerapp:launch_new_instance', 'ipcluster%s = IPython.parallel.apps.ipclusterapp:launch_new_instance', - 'iptest%s = IPython.testing.iptest:main', + 'iptest%s = IPython.testing.iptestcontroller:main', 'irunner%s = IPython.lib.irunner:main', ]] gui_scripts = [] @@ -474,3 +475,26 @@ def require_submodules(command): sys.exit(1) command.run(self) return DecoratedCommand + +#--------------------------------------------------------------------------- +# Notebook related +#--------------------------------------------------------------------------- + +class CompileCSS(Command): + """Recompile Notebook CSS + + Regenerate the compiled CSS from LESS sources. + + Requires various dev dependencies, such as fabric and lessc. + """ + description = "Recompile Notebook CSS" + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + call("fab css", shell=True, cwd=pjoin(repo_root, "IPython", "html")) diff --git a/tools/backport_pr.py b/tools/backport_pr.py index de7f731..69c9cf4 100755 --- a/tools/backport_pr.py +++ b/tools/backport_pr.py @@ -2,24 +2,37 @@ """ Backport pull requests to a particular branch. -Usage: backport_pr.py branch PR +Usage: backport_pr.py branch [PR] e.g.: - backport_pr.py 0.13.1 123 + python tools/backport_pr.py 0.13.1 123 to backport PR #123 onto branch 0.13.1 +or + + python tools/backport_pr.py 1.x + +to see what PRs are marked for backport that have yet to be applied. + """ from __future__ import print_function import os +import re import sys + from subprocess import Popen, PIPE, check_call, check_output from urllib import urlopen -from gh_api import get_pull_request, get_pull_request_files +from gh_api import ( + get_issues_list, + get_pull_request, + get_pull_request_files, + is_pull_request, +) def find_rejects(root='.'): for dirname, dirs, files in os.walk(root): @@ -70,25 +83,61 @@ def backport_pr(branch, num, project='ipython/ipython'): filenames = [ f['filename'] for f in files ] - commit = Popen(['git', 'commit', '-m', msg] + filenames) - commit.wait() - if commit.returncode: - print("commit failed!") - return 1 - else: - print("PR #%i applied, with msg:" % num) - print() - print(msg) - print() + check_call(['git', 'add'] + filenames) + + check_call(['git', 'commit', '-m', msg]) + + print("PR #%i applied, with msg:" % num) + print() + print(msg) + print() if branch != current_branch: check_call(['git', 'checkout', current_branch]) return 0 +backport_re = re.compile(r"[Bb]ackport.*?(\d+)") + +def already_backported(branch, since_tag=None): + """return set of PRs that have been backported already""" + if since_tag is None: + since_tag = check_output(['git','describe', branch, '--abbrev=0']).decode('utf8').strip() + cmd = ['git', 'log', '%s..%s' % (since_tag, branch), '--oneline'] + lines = check_output(cmd).decode('utf8') + return set(int(num) for num in backport_re.findall(lines)) + +def should_backport(labels): + """return set of PRs marked for backport""" + issues = get_issues_list("ipython/ipython", + labels=labels, + state='closed', + auth=True, + ) + should_backport = set() + for issue in issues: + if not is_pull_request(issue): + continue + pr = get_pull_request("ipython/ipython", issue['number'], auth=True) + if not pr['merged']: + print ("Marked PR closed without merge: %i" % pr['number']) + continue + should_backport.add(pr['number']) + return should_backport + if __name__ == '__main__': - if len(sys.argv) < 3: + + if len(sys.argv) < 2: print(__doc__) sys.exit(1) + if len(sys.argv) < 3: + branch = sys.argv[1] + already = already_backported(branch) + should = should_backport("backport-1.2") + print ("The following PRs should be backported:") + for pr in should.difference(already): + print (pr) + sys.exit(0) + sys.exit(backport_pr(sys.argv[1], int(sys.argv[2]))) diff --git a/tools/gh_api.py b/tools/gh_api.py index 8cd798b..22d0bed 100644 --- a/tools/gh_api.py +++ b/tools/gh_api.py @@ -119,12 +119,13 @@ def get_pull_request_files(project, num, auth=False): element_pat = re.compile(r'<(.+?)>') rel_pat = re.compile(r'rel=[\'"](\w+)[\'"]') -def get_paged_request(url, headers=None): +def get_paged_request(url, headers=None, **params): """get a full list, handling APIv3's paging""" results = [] + params.setdefault("per_page", 100) while True: - print("fetching %s" % url, file=sys.stderr) - response = requests.get(url, headers=headers) + print("fetching %s with %s" % (url, params), file=sys.stderr) + response = requests.get(url, headers=headers, params=params) response.raise_for_status() results.extend(response.json()) if 'next' in response.links: @@ -133,28 +134,32 @@ def get_paged_request(url, headers=None): break return results -def get_pulls_list(project, state="closed", auth=False): - """get pull request list - """ - url = "https://api.github.com/repos/{project}/pulls?state={state}&per_page=100".format(project=project, state=state) +def get_pulls_list(project, auth=False, **params): + """get pull request list""" + params.setdefault("state", "closed") + url = "https://api.github.com/repos/{project}/pulls".format(project=project) if auth: headers = make_auth_header() else: headers = None - pages = get_paged_request(url, headers=headers) + pages = get_paged_request(url, headers=headers, params=params) return pages -def get_issues_list(project, state="closed", auth=False): - """get pull request list - """ - url = "https://api.github.com/repos/{project}/pulls?state={state}&per_page=100".format(project=project, state=state) +def get_issues_list(project, auth=False, **params): + """get issues list""" + params.setdefault("state", "closed") + url = "https://api.github.com/repos/{project}/issues".format(project=project) if auth: headers = make_auth_header() else: headers = None - pages = get_paged_request(url, headers=headers) + pages = get_paged_request(url, headers=headers, **params) return pages +def is_pull_request(issue): + """Return True if the given issue is a pull request.""" + return bool(issue.get('pull_request', {}).get('html_url', None)) + # encode_multipart_formdata is from urllib3.filepost # The only change is to iter_fields, to enforce S3's required key ordering diff --git a/tools/github_stats.py b/tools/github_stats.py index cc1072b..fdbcd90 100755 --- a/tools/github_stats.py +++ b/tools/github_stats.py @@ -13,7 +13,7 @@ import sys from datetime import datetime, timedelta from subprocess import check_output -from gh_api import get_paged_request, make_auth_header, get_pull_request +from gh_api import get_paged_request, make_auth_header, get_pull_request, is_pull_request #----------------------------------------------------------------------------- # Globals @@ -50,12 +50,6 @@ def issues2dict(issues): idict[i['number']] = i return idict - -def is_pull_request(issue): - """Return True if the given issue is a pull request.""" - return bool(issue.get('pull_request', {}).get('html_url', None)) - - def split_pulls(all_issues, project="ipython/ipython"): """split a list of closed issues into non-PR Issues and Pull Requests""" pulls = [] diff --git a/tools/release b/tools/release index 4059050..bca0a7d 100755 --- a/tools/release +++ b/tools/release @@ -55,6 +55,8 @@ cd(distdir) print( 'Uploading distribution files...') for fname in os.listdir('.'): + # GitHub doesn't have an API for uploads at the moment + continue print('uploading %s to GitHub' % fname) desc = "IPython %s source distribution" % version post_download("ipython/ipython", fname, description=desc) diff --git a/tox.ini b/tox.ini index 0fdf8e7..4a63c24 100644 --- a/tox.ini +++ b/tox.ini @@ -4,12 +4,15 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py27, py32 +envlist = py27, py33 [testenv] -deps = - nose - +deps = + nose + tornado + jinja2 + sphinx + pygments # To avoid loading IPython module in the current directory, change # current directory to ".tox/py*/tmp" before running test. changedir = {envtmpdir} @@ -18,9 +21,9 @@ commands = # As pip does not treat egg, use easy_install to install PyZMQ. # See also: https://github.com/zeromq/pyzmq easy_install -q pyzmq - iptest [] + iptest --all -[testenv:py32] +[testenv:py33] commands = easy_install -q pyzmq - iptest3 [] + iptest3 --all