Merge pull request #13213 from Kojoley/fix-bunch-of-doctests

Fix bunch of doctests

IPython/lib/display.py

@@ -1,672 +1,674 b''
 """Various display related classes.
 
 Authors : MinRK, gregcaporaso, dannystaple
 """
 from html import escape as html_escape
 from os.path import exists, isfile, splitext, abspath, join, isdir
 from os import walk, sep, fsdecode
 
 from IPython.core.display import DisplayObject, TextDisplayObject
 
 from typing import Tuple, Iterable
 
 __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
            'FileLink', 'FileLinks', 'Code']
 
 
 class Audio(DisplayObject):
     """Create an audio object.
 
     When this object is returned by an input cell or passed to the
     display function, it will result in Audio controls being displayed
     in the frontend (only works in the notebook).
 
     Parameters
     ----------
     data : numpy array, list, unicode, str or bytes
         Can be one of
 
           * Numpy 1d array containing the desired waveform (mono)
           * Numpy 2d array containing waveforms for each channel.
             Shape=(NCHAN, NSAMPLES). For the standard channel order, see
             http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
           * List of float or integer representing the waveform (mono)
           * String containing the filename
           * Bytestring containing raw PCM data or
           * URL pointing to a file on the web.
 
         If the array option is used, the waveform will be normalized.
 
         If a filename or url is used, the format support will be browser
         dependent.
     url : unicode
         A URL to download the data from.
     filename : unicode
         Path to a local file to load the data from.
     embed : boolean
         Should the audio data be embedded using a data URI (True) or should
         the original source be referenced. Set this to True if you want the
         audio to playable later with no internet connection in the notebook.
 
         Default is `True`, unless the keyword argument `url` is set, then
         default value is `False`.
     rate : integer
         The sampling rate of the raw data.
         Only required when data parameter is being used as an array
     autoplay : bool
         Set to True if the audio should immediately start playing.
         Default is `False`.
     normalize : bool
         Whether audio should be normalized (rescaled) to the maximum possible
         range. Default is `True`. When set to `False`, `data` must be between
         -1 and 1 (inclusive), otherwise an error is raised.
         Applies only when `data` is a list or array of samples; other types of
         audio are never normalized.
 
     Examples
     --------
 
     Generate a sound
 
     >>> import numpy as np
-    ... framerate = 44100
-    ... t = np.linspace(0,5,framerate*5)
-    ... data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
-    ... Audio(data, rate=framerate)
+    >>> framerate = 44100
+    >>> t = np.linspace(0,5,framerate*5)
+    >>> data = np.sin(2*np.pi*220*t) + np.sin(2*np.pi*224*t)
+    >>> Audio(data, rate=framerate)
+    <IPython.lib.display.Audio object>
 
     Can also do stereo or more channels
 
     >>> dataleft = np.sin(2*np.pi*220*t)
-    ... dataright = np.sin(2*np.pi*224*t)
-    ... Audio([dataleft, dataright], rate=framerate)
+    >>> dataright = np.sin(2*np.pi*224*t)
+    >>> Audio([dataleft, dataright], rate=framerate)
+    <IPython.lib.display.Audio object>
 
     From URL:
 
-    >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav")
-    >>> Audio(url="http://www.w3schools.com/html/horse.ogg")
+    >>> Audio("http://www.nch.com.au/acm/8k16bitpcm.wav")  # doctest: +SKIP
+    >>> Audio(url="http://www.w3schools.com/html/horse.ogg")  # doctest: +SKIP
 
     From a File:
 
-    >>> Audio('/path/to/sound.wav')
-    >>> Audio(filename='/path/to/sound.ogg')
+    >>> Audio('/path/to/sound.wav')  # doctest: +SKIP
+    >>> Audio(filename='/path/to/sound.ogg')  # doctest: +SKIP
 
     From Bytes:
 
-    >>> Audio(b'RAW_WAV_DATA..')
-    >>> Audio(data=b'RAW_WAV_DATA..')
+    >>> Audio(b'RAW_WAV_DATA..')  # doctest: +SKIP
+    >>> Audio(data=b'RAW_WAV_DATA..')  # doctest: +SKIP
 
     See Also
     --------
     ipywidgets.Audio
     
          AUdio widget with more more flexibility and options.
     
     """
     _read_flags = 'rb'
 
     def __init__(self, data=None, filename=None, url=None, embed=None, rate=None, autoplay=False, normalize=True, *,
                  element_id=None):
         if filename is None and url is None and data is None:
             raise ValueError("No audio data found. Expecting filename, url, or data.")
         if embed is False and url is None:
             raise ValueError("No url found. Expecting url when embed=False")
 
         if url is not None and embed is not True:
             self.embed = False
         else:
             self.embed = True
         self.autoplay = autoplay
         self.element_id = element_id
         super(Audio, self).__init__(data=data, url=url, filename=filename)
 
         if self.data is not None and not isinstance(self.data, bytes):
             if rate is None:
                 raise ValueError("rate must be specified when data is a numpy array or list of audio samples.")
             self.data = Audio._make_wav(data, rate, normalize)
 
     def reload(self):
         """Reload the raw data from file or URL."""
         import mimetypes
         if self.embed:
             super(Audio, self).reload()
 
         if self.filename is not None:
             self.mimetype = mimetypes.guess_type(self.filename)[0]
         elif self.url is not None:
             self.mimetype = mimetypes.guess_type(self.url)[0]
         else:
             self.mimetype = "audio/wav"
 
     @staticmethod
     def _make_wav(data, rate, normalize):
         """ Transform a numpy array to a PCM bytestring """
         from io import BytesIO
         import wave
 
         try:
             scaled, nchan = Audio._validate_and_normalize_with_numpy(data, normalize)
         except ImportError:
             scaled, nchan = Audio._validate_and_normalize_without_numpy(data, normalize)
 
         fp = BytesIO()
         waveobj = wave.open(fp,mode='wb')
         waveobj.setnchannels(nchan)
         waveobj.setframerate(rate)
         waveobj.setsampwidth(2)
         waveobj.setcomptype('NONE','NONE')
         waveobj.writeframes(scaled)
         val = fp.getvalue()
         waveobj.close()
 
         return val
 
     @staticmethod
     def _validate_and_normalize_with_numpy(data, normalize) -> Tuple[bytes, int]:
         import numpy as np
 
         data = np.array(data, dtype=float)
         if len(data.shape) == 1:
             nchan = 1
         elif len(data.shape) == 2:
             # In wave files,channels are interleaved. E.g.,
             # "L1R1L2R2..." for stereo. See
             # http://msdn.microsoft.com/en-us/library/windows/hardware/dn653308(v=vs.85).aspx
             # for channel ordering
             nchan = data.shape[0]
             data = data.T.ravel()
         else:
             raise ValueError('Array audio input must be a 1D or 2D array')
         
         max_abs_value = np.max(np.abs(data))
         normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
         scaled = data / normalization_factor * 32767
         return scaled.astype("<h").tobytes(), nchan
 
     @staticmethod
     def _validate_and_normalize_without_numpy(data, normalize):
         import array
         import sys
 
         data = array.array('f', data)
 
         try:
             max_abs_value = float(max([abs(x) for x in data]))
         except TypeError as e:
             raise TypeError('Only lists of mono audio are '
                 'supported if numpy is not installed') from e
 
         normalization_factor = Audio._get_normalization_factor(max_abs_value, normalize)
         scaled = array.array('h', [int(x / normalization_factor * 32767) for x in data])
         if sys.byteorder == 'big':
             scaled.byteswap()
         nchan = 1
         return scaled.tobytes(), nchan
 
     @staticmethod
     def _get_normalization_factor(max_abs_value, normalize):
         if not normalize and max_abs_value > 1:
             raise ValueError('Audio data must be between -1 and 1 when normalize=False.')
         return max_abs_value if normalize else 1
 
     def _data_and_metadata(self):
         """shortcut for returning metadata with url information, if defined"""
         md = {}
         if self.url:
             md['url'] = self.url
         if md:
             return self.data, md
         else:
             return self.data
 
     def _repr_html_(self):
         src = """
                 <audio {element_id} controls="controls" {autoplay}>
                     <source src="{src}" type="{type}" />
                     Your browser does not support the audio element.
                 </audio>
               """
         return src.format(src=self.src_attr(), type=self.mimetype, autoplay=self.autoplay_attr(),
                           element_id=self.element_id_attr())
 
     def src_attr(self):
         import base64
         if self.embed and (self.data is not None):
             data = base64=base64.b64encode(self.data).decode('ascii')
             return """data:{type};base64,{base64}""".format(type=self.mimetype,
                                                             base64=data)
         elif self.url is not None:
             return self.url
         else:
             return ""
 
     def autoplay_attr(self):
         if(self.autoplay):
             return 'autoplay="autoplay"'
         else:
             return ''
     
     def element_id_attr(self):
         if (self.element_id):
             return 'id="{element_id}"'.format(element_id=self.element_id)
         else:
             return ''
 
 class IFrame(object):
     """
     Generic class to embed an iframe in an IPython notebook
     """
 
     iframe = """
         <iframe
             width="{width}"
             height="{height}"
             src="{src}{params}"
             frameborder="0"
             allowfullscreen
             {extras}
         ></iframe>
         """
 
     def __init__(self, src, width, height, extras: Iterable[str] = None, **kwargs):
         if extras is None:
             extras = []
 
         self.src = src
         self.width = width
         self.height = height
         self.extras = extras
         self.params = kwargs
 
     def _repr_html_(self):
         """return the embed iframe"""
         if self.params:
             from urllib.parse import urlencode
             params = "?" + urlencode(self.params)
         else:
             params = ""
         return self.iframe.format(
             src=self.src,
             width=self.width,
             height=self.height,
             params=params,
             extras=" ".join(self.extras),
         )
 
 
 class YouTubeVideo(IFrame):
     """Class for embedding a YouTube Video in an IPython session, based on its video id.
 
     e.g. to embed the video from https://www.youtube.com/watch?v=foo , you would
     do::
 
         vid = YouTubeVideo("foo")
         display(vid)
 
     To start from 30 seconds::
 
         vid = YouTubeVideo("abc", start=30)
         display(vid)
 
     To calculate seconds from time as hours, minutes, seconds use
     :class:`datetime.timedelta`::
 
         start=int(timedelta(hours=1, minutes=46, seconds=40).total_seconds())
 
     Other parameters can be provided as documented at
     https://developers.google.com/youtube/player_parameters#Parameters
     
     When converting the notebook using nbconvert, a jpeg representation of the video
     will be inserted in the document.
     """
 
     def __init__(self, id, width=400, height=300, allow_autoplay=False, **kwargs):
         self.id=id
         src = "https://www.youtube.com/embed/{0}".format(id)
         if allow_autoplay:
             extras = list(kwargs.get("extras", [])) + ['allow="autoplay"']
             kwargs.update(autoplay=1, extras=extras)
         super(YouTubeVideo, self).__init__(src, width, height, **kwargs)
 
     def _repr_jpeg_(self):
         # Deferred import
         from urllib.request import urlopen
 
         try:
             return urlopen("https://img.youtube.com/vi/{id}/hqdefault.jpg".format(id=self.id)).read()
         except IOError:
             return None
 
 class VimeoVideo(IFrame):
     """
     Class for embedding a Vimeo video in an IPython session, based on its video id.
     """
 
     def __init__(self, id, width=400, height=300, **kwargs):
         src="https://player.vimeo.com/video/{0}".format(id)
         super(VimeoVideo, self).__init__(src, width, height, **kwargs)
 
 class ScribdDocument(IFrame):
     """
     Class for embedding a Scribd document in an IPython session
 
     Use the start_page params to specify a starting point in the document
     Use the view_mode params to specify display type one off scroll | slideshow | book
 
     e.g to Display Wes' foundational paper about PANDAS in book mode from page 3
 
     ScribdDocument(71048089, width=800, height=400, start_page=3, view_mode="book")
     """
 
     def __init__(self, id, width=400, height=300, **kwargs):
         src="https://www.scribd.com/embeds/{0}/content".format(id)
         super(ScribdDocument, self).__init__(src, width, height, **kwargs)
 
 class FileLink(object):
     """Class for embedding a local file link in an IPython session, based on path
 
     e.g. to embed a link that was generated in the IPython notebook as my/data.txt
 
     you would do::
 
         local_file = FileLink("my/data.txt")
         display(local_file)
 
     or in the HTML notebook, just::
 
         FileLink("my/data.txt")
     """
 
     html_link_str = "<a href='%s' target='_blank'>%s</a>"
 
     def __init__(self,
                  path,
                  url_prefix='',
                  result_html_prefix='',
                  result_html_suffix='<br>'):
         """
         Parameters
         ----------
         path : str
             path to the file or directory that should be formatted
         url_prefix : str
             prefix to be prepended to all files to form a working link [default:
             '']
         result_html_prefix : str
             text to append to beginning to link [default: '']
         result_html_suffix : str
             text to append at the end of link [default: '<br>']
         """
         if isdir(path):
             raise ValueError("Cannot display a directory using FileLink. "
               "Use FileLinks to display '%s'." % path)
         self.path = fsdecode(path)
         self.url_prefix = url_prefix
         self.result_html_prefix = result_html_prefix
         self.result_html_suffix = result_html_suffix
 
     def _format_path(self):
         fp = ''.join([self.url_prefix, html_escape(self.path)])
         return ''.join([self.result_html_prefix,
                         self.html_link_str % \
                             (fp, html_escape(self.path, quote=False)),
                         self.result_html_suffix])
 
     def _repr_html_(self):
         """return html link to file
         """
         if not exists(self.path):
             return ("Path (<tt>%s</tt>) doesn't exist. "
                     "It may still be in the process of "
                     "being generated, or you may have the "
                     "incorrect path." % self.path)
 
         return self._format_path()
 
     def __repr__(self):
         """return absolute path to file
         """
         return abspath(self.path)
 
 class FileLinks(FileLink):
     """Class for embedding local file links in an IPython session, based on path
 
     e.g. to embed links to files that were generated in the IPython notebook
     under ``my/data``, you would do::
 
         local_files = FileLinks("my/data")
         display(local_files)
 
     or in the HTML notebook, just::
 
         FileLinks("my/data")
     """
     def __init__(self,
                  path,
                  url_prefix='',
                  included_suffixes=None,
                  result_html_prefix='',
                  result_html_suffix='<br>',
                  notebook_display_formatter=None,
                  terminal_display_formatter=None,
                  recursive=True):
         """
         See :class:`FileLink` for the ``path``, ``url_prefix``,
         ``result_html_prefix`` and ``result_html_suffix`` parameters.
 
         included_suffixes : list
           Filename suffixes to include when formatting output [default: include
           all files]
 
         notebook_display_formatter : function
           Used to format links for display in the notebook. See discussion of
           formatter functions below.
 
         terminal_display_formatter : function
           Used to format links for display in the terminal. See discussion of
           formatter functions below.
 
         Formatter functions must be of the form::
 
             f(dirname, fnames, included_suffixes)
 
         dirname : str
           The name of a directory
         fnames : list
           The files in that directory
         included_suffixes : list
           The file suffixes that should be included in the output (passing None
           meansto include all suffixes in the output in the built-in formatters)
         recursive : boolean
           Whether to recurse into subdirectories. Default is True.
 
         The function should return a list of lines that will be printed in the
         notebook (if passing notebook_display_formatter) or the terminal (if
         passing terminal_display_formatter). This function is iterated over for
         each directory in self.path. Default formatters are in place, can be
         passed here to support alternative formatting.
 
         """
         if isfile(path):
             raise ValueError("Cannot display a file using FileLinks. "
               "Use FileLink to display '%s'." % path)
         self.included_suffixes = included_suffixes
         # remove trailing slashes for more consistent output formatting
         path = path.rstrip('/')
 
         self.path = path
         self.url_prefix = url_prefix
         self.result_html_prefix = result_html_prefix
         self.result_html_suffix = result_html_suffix
 
         self.notebook_display_formatter = \
              notebook_display_formatter or self._get_notebook_display_formatter()
         self.terminal_display_formatter = \
              terminal_display_formatter or self._get_terminal_display_formatter()
 
         self.recursive = recursive
 
     def _get_display_formatter(self,
                                dirname_output_format,
                                fname_output_format,
                                fp_format,
                                fp_cleaner=None):
         """ generate built-in formatter function
 
            this is used to define both the notebook and terminal built-in
             formatters as they only differ by some wrapper text for each entry
 
            dirname_output_format: string to use for formatting directory
             names, dirname will be substituted for a single "%s" which
             must appear in this string
            fname_output_format: string to use for formatting file names,
             if a single "%s" appears in the string, fname will be substituted
             if two "%s" appear in the string, the path to fname will be
              substituted for the first and fname will be substituted for the
              second
            fp_format: string to use for formatting filepaths, must contain
             exactly two "%s" and the dirname will be substituted for the first
             and fname will be substituted for the second
         """
         def f(dirname, fnames, included_suffixes=None):
             result = []
             # begin by figuring out which filenames, if any,
             # are going to be displayed
             display_fnames = []
             for fname in fnames:
                 if (isfile(join(dirname,fname)) and
                        (included_suffixes is None or
                         splitext(fname)[1] in included_suffixes)):
                       display_fnames.append(fname)
 
             if len(display_fnames) == 0:
                 # if there are no filenames to display, don't print anything
                 # (not even the directory name)
                 pass
             else:
                 # otherwise print the formatted directory name followed by
                 # the formatted filenames
                 dirname_output_line = dirname_output_format % dirname
                 result.append(dirname_output_line)
                 for fname in display_fnames:
                     fp = fp_format % (dirname,fname)
                     if fp_cleaner is not None:
                         fp = fp_cleaner(fp)
                     try:
                         # output can include both a filepath and a filename...
                         fname_output_line = fname_output_format % (fp, fname)
                     except TypeError:
                         # ... or just a single filepath
                         fname_output_line = fname_output_format % fname
                     result.append(fname_output_line)
             return result
         return f
 
     def _get_notebook_display_formatter(self,
                                         spacer="&nbsp;&nbsp;"):
         """ generate function to use for notebook formatting
         """
         dirname_output_format = \
          self.result_html_prefix + "%s/" + self.result_html_suffix
         fname_output_format = \
          self.result_html_prefix + spacer + self.html_link_str + self.result_html_suffix
         fp_format = self.url_prefix + '%s/%s'
         if sep == "\\":
             # Working on a platform where the path separator is "\", so
             # must convert these to "/" for generating a URI
             def fp_cleaner(fp):
                 # Replace all occurrences of backslash ("\") with a forward
                 # slash ("/") - this is necessary on windows when a path is
                 # provided as input, but we must link to a URI
                 return fp.replace('\\','/')
         else:
             fp_cleaner = None
 
         return self._get_display_formatter(dirname_output_format,
                                            fname_output_format,
                                            fp_format,
                                            fp_cleaner)
 
     def _get_terminal_display_formatter(self,
                                         spacer="  "):
         """ generate function to use for terminal formatting
         """
         dirname_output_format = "%s/"
         fname_output_format = spacer + "%s"
         fp_format = '%s/%s'
 
         return self._get_display_formatter(dirname_output_format,
                                            fname_output_format,
                                            fp_format)
 
     def _format_path(self):
         result_lines = []
         if self.recursive:
             walked_dir = list(walk(self.path))
         else:
             walked_dir = [next(walk(self.path))]
         walked_dir.sort()
         for dirname, subdirs, fnames in walked_dir:
             result_lines += self.notebook_display_formatter(dirname, fnames, self.included_suffixes)
         return '\n'.join(result_lines)
 
     def __repr__(self):
         """return newline-separated absolute paths
         """
         result_lines = []
         if self.recursive:
             walked_dir = list(walk(self.path))
         else:
             walked_dir = [next(walk(self.path))]
         walked_dir.sort()
         for dirname, subdirs, fnames in walked_dir:
             result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
         return '\n'.join(result_lines)
 
 
 class Code(TextDisplayObject):
     """Display syntax-highlighted source code.
 
     This uses Pygments to highlight the code for HTML and Latex output.
 
     Parameters
     ----------
     data : str
         The code as a string
     url : str
         A URL to fetch the code from
     filename : str
         A local filename to load the code from
     language : str
         The short name of a Pygments lexer to use for highlighting.
         If not specified, it will guess the lexer based on the filename
         or the code. Available lexers: http://pygments.org/docs/lexers/
     """
     def __init__(self, data=None, url=None, filename=None, language=None):
         self.language = language
         super().__init__(data=data, url=url, filename=filename)
 
     def _get_lexer(self):
         if self.language:
             from pygments.lexers import get_lexer_by_name
             return get_lexer_by_name(self.language)
         elif self.filename:
             from pygments.lexers import get_lexer_for_filename
             return get_lexer_for_filename(self.filename)
         else:
             from pygments.lexers import guess_lexer
             return guess_lexer(self.data)
 
     def __repr__(self):
         return self.data
 
     def _repr_html_(self):
         from pygments import highlight
         from pygments.formatters import HtmlFormatter
         fmt = HtmlFormatter()
         style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
         return style + highlight(self.data, self._get_lexer(), fmt)
 
     def _repr_latex_(self):
         from pygments import highlight
         from pygments.formatters import LatexFormatter
         return highlight(self.data, self._get_lexer(), LatexFormatter())

IPython/lib/lexers.py

@@ -1,532 +1,526 b''
 # -*- coding: utf-8 -*-
 """
 Defines a variety of Pygments lexers for highlighting IPython code.
 
 This includes:
 
     IPythonLexer, IPython3Lexer
         Lexers for pure IPython (python + magic/shell commands)
 
     IPythonPartialTracebackLexer, IPythonTracebackLexer
         Supports 2.x and 3.x via keyword `python3`.  The partial traceback
         lexer reads everything but the Python code appearing in a traceback.
         The full lexer combines the partial lexer with an IPython lexer.
 
     IPythonConsoleLexer
         A lexer for IPython console sessions, with support for tracebacks.
 
     IPyLexer
         A friendly lexer which examines the first line of text and from it,
         decides whether to use an IPython lexer or an IPython console lexer.
         This is probably the only lexer that needs to be explicitly added
         to Pygments.
 
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 # Standard library
 import re
 
 # Third party
 from pygments.lexers import (
     BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
     Python3Lexer, TexLexer)
 from pygments.lexer import (
     Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
 )
 from pygments.token import (
     Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
 )
 from pygments.util import get_bool_opt
 
 # Local
 
 line_re = re.compile('.*?\n')
 
 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
            'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
            'IPythonConsoleLexer', 'IPyLexer']
 
 
 def build_ipy_lexer(python3):
     """Builds IPython lexers depending on the value of `python3`.
 
     The lexer inherits from an appropriate Python lexer and then adds
     information about IPython specific keywords (i.e. magic commands,
     shell commands, etc.)
 
     Parameters
     ----------
     python3 : bool
         If `True`, then build an IPython lexer from a Python 3 lexer.
 
     """
     # It would be nice to have a single IPython lexer class which takes
     # a boolean `python3`.  But since there are two Python lexer classes,
     # we will also have two IPython lexer classes.
     if python3:
         PyLexer = Python3Lexer
         name = 'IPython3'
         aliases = ['ipython3']
         doc = """IPython3 Lexer"""
     else:
         PyLexer = PythonLexer
         name = 'IPython'
         aliases = ['ipython2', 'ipython']
         doc = """IPython Lexer"""
 
     ipython_tokens = [
        (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
         (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
         (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
         (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
         (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
         (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
         (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
         (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
         (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
         (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
         (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
         (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
         (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
                                                 using(BashLexer), Text)),
         (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
         (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
         (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
         (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
         (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
     ]
 
     tokens = PyLexer.tokens.copy()
     tokens['root'] = ipython_tokens + tokens['root']
 
     attrs = {'name': name, 'aliases': aliases, 'filenames': [],
              '__doc__': doc, 'tokens': tokens}
 
     return type(name, (PyLexer,), attrs)
 
 
 IPython3Lexer = build_ipy_lexer(python3=True)
 IPythonLexer = build_ipy_lexer(python3=False)
 
 
 class IPythonPartialTracebackLexer(RegexLexer):
     """
     Partial lexer for IPython tracebacks.
 
     Handles all the non-python output.
 
     """
     name = 'IPython Partial Traceback'
 
     tokens = {
         'root': [
             # Tracebacks for syntax errors have a different style.
             # For both types of tracebacks, we mark the first line with
             # Generic.Traceback.  For syntax errors, we mark the filename
             # as we mark the filenames for non-syntax tracebacks.
             #
             # These two regexps define how IPythonConsoleLexer finds a
             # traceback.
             #
             ## Non-syntax traceback
             (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
             ## Syntax traceback
             (r'^(  File)(.*)(, line )(\d+\n)',
              bygroups(Generic.Traceback, Name.Namespace,
                       Generic.Traceback, Literal.Number.Integer)),
 
             # (Exception Identifier)(Whitespace)(Traceback Message)
             (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
              bygroups(Name.Exception, Generic.Whitespace, Text)),
             # (Module/Filename)(Text)(Callee)(Function Signature)
             # Better options for callee and function signature?
             (r'(.*)( in )(.*)(\(.*\)\n)',
              bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
             # Regular line: (Whitespace)(Line Number)(Python Code)
             (r'(\s*?)(\d+)(.*?\n)',
              bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
             # Emphasized line: (Arrow)(Line Number)(Python Code)
             # Using Exception token so arrow color matches the Exception.
             (r'(-*>?\s?)(\d+)(.*?\n)',
              bygroups(Name.Exception, Literal.Number.Integer, Other)),
             # (Exception Identifier)(Message)
             (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
              bygroups(Name.Exception, Text)),
             # Tag everything else as Other, will be handled later.
             (r'.*\n', Other),
         ],
     }
 
 
 class IPythonTracebackLexer(DelegatingLexer):
     """
     IPython traceback lexer.
 
     For doctests, the tracebacks can be snipped as much as desired with the
     exception to the lines that designate a traceback. For non-syntax error
     tracebacks, this is the line of hyphens. For syntax error tracebacks,
     this is the line which lists the File and line number.
 
     """
     # The lexer inherits from DelegatingLexer.  The "root" lexer is an
     # appropriate IPython lexer, which depends on the value of the boolean
     # `python3`.  First, we parse with the partial IPython traceback lexer.
     # Then, any code marked with the "Other" token is delegated to the root
     # lexer.
     #
     name = 'IPython Traceback'
     aliases = ['ipythontb']
 
     def __init__(self, **options):
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipython3tb']
         else:
             self.aliases = ['ipython2tb', 'ipythontb']
 
         if self.python3:
             IPyLexer = IPython3Lexer
         else:
             IPyLexer = IPythonLexer
 
         DelegatingLexer.__init__(self, IPyLexer,
                                  IPythonPartialTracebackLexer, **options)
 
 class IPythonConsoleLexer(Lexer):
     """
     An IPython console lexer for IPython code-blocks and doctests, such as:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: a = 'foo'
 
             In [2]: a
             Out[2]: 'foo'
 
-            In [3]: print a
+            In [3]: print(a)
             foo
 
-            In [4]: 1 / 0
-
 
     Support is also provided for IPython exceptions:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: raise Exception
-
-            ---------------------------------------------------------------------------
-            Exception                                 Traceback (most recent call last)
-            <ipython-input-1-fca2ab0ca76b> in <module>
-            ----> 1 raise Exception
-
-            Exception:
+            Traceback (most recent call last):
+            ...
+            Exception
 
     """
     name = 'IPython console session'
     aliases = ['ipythonconsole']
     mimetypes = ['text/x-ipython-console']
 
     # The regexps used to determine what is input and what is output.
     # The default prompts for IPython are:
     #
     #    in           = 'In [#]: '
     #    continuation = '   .D.: '
     #    template     = 'Out[#]: '
     #
     # Where '#' is the 'prompt number' or 'execution count' and 'D' 
     # D is a number of dots  matching the width of the execution count 
     #
     in1_regex = r'In \[[0-9]+\]: '
     in2_regex = r'   \.\.+\.: '
     out_regex = r'Out\[[0-9]+\]: '
 
     #: The regex to determine when a traceback starts.
     ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
 
     def __init__(self, **options):
         """Initialize the IPython console lexer.
 
         Parameters
         ----------
         python3 : bool
             If `True`, then the console inputs are parsed using a Python 3
             lexer. Otherwise, they are parsed using a Python 2 lexer.
         in1_regex : RegexObject
             The compiled regular expression used to detect the start
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         in2_regex : RegexObject
             The compiled regular expression used to detect the continuation
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         out_regex : RegexObject
             The compiled regular expression used to detect outputs. If `None`,
             then the default output prompt is assumed.
 
         """
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipython3console']
         else:
             self.aliases = ['ipython2console', 'ipythonconsole']
 
         in1_regex = options.get('in1_regex', self.in1_regex)
         in2_regex = options.get('in2_regex', self.in2_regex)
         out_regex = options.get('out_regex', self.out_regex)
 
         # So that we can work with input and output prompts which have been
         # rstrip'd (possibly by editors) we also need rstrip'd variants. If
         # we do not do this, then such prompts will be tagged as 'output'.
         # The reason can't just use the rstrip'd variants instead is because
         # we want any whitespace associated with the prompt to be inserted
         # with the token. This allows formatted code to be modified so as hide
         # the appearance of prompts, with the whitespace included. One example
         # use of this is in copybutton.js from the standard lib Python docs.
         in1_regex_rstrip = in1_regex.rstrip() + '\n'
         in2_regex_rstrip = in2_regex.rstrip() + '\n'
         out_regex_rstrip = out_regex.rstrip() + '\n'
 
         # Compile and save them all.
         attrs = ['in1_regex', 'in2_regex', 'out_regex',
                  'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
         for attr in attrs:
             self.__setattr__(attr, re.compile(locals()[attr]))
 
         Lexer.__init__(self, **options)
 
         if self.python3:
             pylexer = IPython3Lexer
             tblexer = IPythonTracebackLexer
         else:
             pylexer = IPythonLexer
             tblexer = IPythonTracebackLexer
 
         self.pylexer = pylexer(**options)
         self.tblexer = tblexer(**options)
 
         self.reset()
 
     def reset(self):
         self.mode = 'output'
         self.index = 0
         self.buffer = u''
         self.insertions = []
 
     def buffered_tokens(self):
         """
         Generator of unprocessed tokens after doing insertions and before
         changing to a new state.
 
         """
         if self.mode == 'output':
             tokens = [(0, Generic.Output, self.buffer)]
         elif self.mode == 'input':
             tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
         else: # traceback
             tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
 
         for i, t, v in do_insertions(self.insertions, tokens):
             # All token indexes are relative to the buffer.
             yield self.index + i, t, v
 
         # Clear it all
         self.index += len(self.buffer)
         self.buffer = u''
         self.insertions = []
 
     def get_mci(self, line):
         """
         Parses the line and returns a 3-tuple: (mode, code, insertion).
 
         `mode` is the next mode (or state) of the lexer, and is always equal
         to 'input', 'output', or 'tb'.
 
         `code` is a portion of the line that should be added to the buffer
         corresponding to the next mode and eventually lexed by another lexer.
         For example, `code` could be Python code if `mode` were 'input'.
 
         `insertion` is a 3-tuple (index, token, text) representing an
         unprocessed "token" that will be inserted into the stream of tokens
         that are created from the buffer once we change modes. This is usually
         the input or output prompt.
 
         In general, the next mode depends on current mode and on the contents
         of `line`.
 
         """
         # To reduce the number of regex match checks, we have multiple
         # 'if' blocks instead of 'if-elif' blocks.
 
         # Check for possible end of input
         in2_match = self.in2_regex.match(line)
         in2_match_rstrip = self.in2_regex_rstrip.match(line)
         if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
            in2_match_rstrip:
             end_input = True
         else:
             end_input = False
         if end_input and self.mode != 'tb':
             # Only look for an end of input when not in tb mode.
             # An ellipsis could appear within the traceback.
             mode = 'output'
             code = u''
             insertion = (0, Generic.Prompt, line)
             return mode, code, insertion
 
         # Check for output prompt
         out_match = self.out_regex.match(line)
         out_match_rstrip = self.out_regex_rstrip.match(line)
         if out_match or out_match_rstrip:
             mode = 'output'
             if out_match:
                 idx = out_match.end()
             else:
                 idx = out_match_rstrip.end()
             code = line[idx:]
             # Use the 'heading' token for output.  We cannot use Generic.Error
             # since it would conflict with exceptions.
             insertion = (0, Generic.Heading, line[:idx])
             return mode, code, insertion
 
 
         # Check for input or continuation prompt (non stripped version)
         in1_match = self.in1_regex.match(line)
         if in1_match or (in2_match and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match:
                 idx = in1_match.end()
             else: # in2_match
                 idx = in2_match.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for input or continuation prompt (stripped version)
         in1_match_rstrip = self.in1_regex_rstrip.match(line)
         if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match_rstrip:
                 idx = in1_match_rstrip.end()
             else: # in2_match
                 idx = in2_match_rstrip.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for traceback
         if self.ipytb_start.match(line):
             mode = 'tb'
             code = line
             insertion = None
             return mode, code, insertion
 
         # All other stuff...
         if self.mode in ('input', 'output'):
             # We assume all other text is output. Multiline input that
             # does not use the continuation marker cannot be detected.
             # For example, the 3 in the following is clearly output:
             #
             #    In [1]: print 3
             #    3
             #
             # But the following second line is part of the input:
             #
             #    In [2]: while True:
             #        print True
             #
             # In both cases, the 2nd line will be 'output'.
             #
             mode = 'output'
         else:
             mode = 'tb'
 
         code = line
         insertion = None
 
         return mode, code, insertion
 
     def get_tokens_unprocessed(self, text):
         self.reset()
         for match in line_re.finditer(text):
             line = match.group()
             mode, code, insertion = self.get_mci(line)
 
             if mode != self.mode:
                 # Yield buffered tokens before transitioning to new mode.
                 for token in self.buffered_tokens():
                     yield token
                 self.mode = mode
 
             if insertion:
                 self.insertions.append((len(self.buffer), [insertion]))
             self.buffer += code
 
         for token in self.buffered_tokens():
             yield token
 
 class IPyLexer(Lexer):
     r"""
     Primary lexer for all IPython-like code.
 
     This is a simple helper lexer.  If the first line of the text begins with
     "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
     lexer. If not, then the entire text is parsed with an IPython lexer.
 
     The goal is to reduce the number of lexers that are registered
     with Pygments.
 
     """
     name = 'IPy session'
     aliases = ['ipy']
 
     def __init__(self, **options):
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipy3']
         else:
             self.aliases = ['ipy2', 'ipy']
 
         Lexer.__init__(self, **options)
 
         self.IPythonLexer = IPythonLexer(**options)
         self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
 
     def get_tokens_unprocessed(self, text):
         # Search for the input prompt anywhere...this allows code blocks to
         # begin with comments as well.
         if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
             lex = self.IPythonConsoleLexer
         else:
             lex = self.IPythonLexer
         for token in lex.get_tokens_unprocessed(text):
             yield token
 

IPython/lib/security.py

@@ -1,114 +1,114 b''
 """
 Password generation for the IPython notebook.
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # Stdlib
 import getpass
 import hashlib
 import random
 
 # Our own
 from IPython.core.error import UsageError
 from IPython.utils.py3compat import encode
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Length of the salt in nr of hex chars, which implies salt_len * 4
 # bits of randomness.
 salt_len = 12
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def passwd(passphrase=None, algorithm='sha1'):
     """Generate hashed password and salt for use in notebook configuration.
 
     In the notebook configuration, set `c.NotebookApp.password` to
     the generated string.
 
     Parameters
     ----------
     passphrase : str
         Password to hash.  If unspecified, the user is asked to input
         and verify a password.
     algorithm : str
         Hashing algorithm to use (e.g, 'sha1' or any argument supported
         by :func:`hashlib.new`).
 
     Returns
     -------
     hashed_passphrase : str
         Hashed password, in the format 'hash_algorithm:salt:passphrase_hash'.
 
     Examples
     --------
     >>> passwd('mypassword')
-    'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12'
+    'sha1:7cf3:b7d6da294ea9592a9480c8f52e63cd42cfb9dd12'  # random
 
     """
     if passphrase is None:
         for i in range(3):
             p0 = getpass.getpass('Enter password: ')
             p1 = getpass.getpass('Verify password: ')
             if p0 == p1:
                 passphrase = p0
                 break
             else:
                 print('Passwords do not match.')
         else:
             raise UsageError('No matching passwords found. Giving up.')
 
     h = hashlib.new(algorithm)
     salt = ('%0' + str(salt_len) + 'x') % random.getrandbits(4 * salt_len)
     h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
 
     return ':'.join((algorithm, salt, h.hexdigest()))
 
 
 def passwd_check(hashed_passphrase, passphrase):
     """Verify that a given passphrase matches its hashed version.
 
     Parameters
     ----------
     hashed_passphrase : str
         Hashed password, in the format returned by `passwd`.
     passphrase : str
         Passphrase to validate.
 
     Returns
     -------
     valid : bool
         True if the passphrase matches the hash.
 
     Examples
     --------
     >>> from IPython.lib.security import passwd_check
     >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
     ...              'mypassword')
     True
 
     >>> passwd_check('sha1:0e112c3ddfce:a68df677475c2b47b6e86d0467eec97ac5f4b85a',
     ...              'anotherpassword')
     False
     """
     try:
         algorithm, salt, pw_digest = hashed_passphrase.split(':', 2)
     except (ValueError, TypeError):
         return False
 
     try:
         h = hashlib.new(algorithm)
     except ValueError:
         return False
 
     if len(pw_digest) == 0:
         return False
 
     h.update(encode(passphrase, 'utf-8') + encode(salt, 'ascii'))
 
     return h.hexdigest() == pw_digest

IPython/testing/plugin/dtexample.py

@@ -1,157 +1,167 b''
 """Simple example using doctests.
 
 This file just contains doctests both using plain python and IPython prompts.
 All tests should be loaded by nose.
 """
 
+import os
+
+
 def pyfunc():
     """Some pure python tests...
 
     >>> pyfunc()
     'pyfunc'
 
     >>> import os
 
     >>> 2+3
     5
 
     >>> for i in range(3):
     ...     print(i, end=' ')
     ...     print(i+1, end=' ')
     ...
     0 1 1 2 2 3 
     """
     return 'pyfunc'
 
 def ipfunc():
     """Some ipython tests...
 
     In [1]: import os
 
     In [3]: 2+3
     Out[3]: 5
 
     In [26]: for i in range(3):
        ....:     print(i, end=' ')
        ....:     print(i+1, end=' ')
        ....:
     0 1 1 2 2 3 
 
 
-    Examples that access the operating system work:
-
-    In [1]: !echo hello
-    hello
-
-    In [2]: !echo hello > /tmp/foo_iptest
-
-    In [3]: !cat /tmp/foo_iptest
-    hello
-
-    In [4]: rm -f /tmp/foo_iptest
-
     It's OK to use '_' for the last result, but do NOT try to use IPython's
     numbered history of _NN outputs, since those won't exist under the
     doctest environment:
 
     In [7]: 'hi'
     Out[7]: 'hi'
 
     In [8]: print(repr(_))
     'hi'
     
     In [7]: 3+4
     Out[7]: 7
 
     In [8]: _+3
     Out[8]: 10
 
     In [9]: ipfunc()
     Out[9]: 'ipfunc'
     """
     return 'ipfunc'
 
 
+def ipos():
+    """Examples that access the operating system work:
+
+    In [1]: !echo hello
+    hello
+
+    In [2]: !echo hello > /tmp/foo_iptest
+
+    In [3]: !cat /tmp/foo_iptest
+    hello
+
+    In [4]: rm -f /tmp/foo_iptest
+    """
+    pass
+
+
+ipos.__skip_doctest__ = os.name == "nt"
+
+
 def ranfunc():
     """A function with some random output.
 
        Normal examples are verified as usual:
        >>> 1+3
        4
 
        But if you put '# random' in the output, it is ignored:
        >>> 1+3
        junk goes here...  # random
 
        >>> 1+2
        again,  anything goes #random
        if multiline, the random mark is only needed once.
 
        >>> 1+2
        You can also put the random marker at the end:
        # random
 
        >>> 1+2
        # random
        .. or at the beginning.
 
        More correct input is properly verified:
        >>> ranfunc()
        'ranfunc'
     """
     return 'ranfunc'
 
 
 def random_all():
     """A function where we ignore the output of ALL examples.
 
     Examples:
 
       # all-random
 
       This mark tells the testing machinery that all subsequent examples should
       be treated as random (ignoring their output).  They are still executed,
       so if a they raise an error, it will be detected as such, but their
       output is completely ignored.
 
       >>> 1+3
       junk goes here...
 
       >>> 1+3
       klasdfj;
 
       >>> 1+2
       again,  anything goes
       blah...
     """
     pass
 
 def iprand():
     """Some ipython tests with random output.
 
     In [7]: 3+4
     Out[7]: 7
 
     In [8]: print('hello')
     world  # random
 
     In [9]: iprand()
     Out[9]: 'iprand'
     """
     return 'iprand'
 
 def iprand_all():
     """Some ipython tests with fully random output.
 
     # all-random
     
     In [7]: 1
     Out[7]: 99
 
     In [8]: print('hello')
     world
 
     In [9]: iprand_all()
     Out[9]: 'junk'
     """
     return 'iprand_all'

IPython/testing/plugin/test_exampleip.txt

@@ -1,30 +1,30 b''
 =================================
  Tests in example form - IPython
 =================================
 
 You can write text files with examples that use IPython prompts (as long as you
 use the nose ipython doctest plugin), but you can not mix and match prompt
 styles in a single file.  That is, you either use all ``>>>`` prompts or all
 IPython-style prompts.  Your test suite *can* have both types, you just need to
 put each type of example in a separate. Using IPython prompts, you can paste
 directly from your session::
 
     In [5]: s="Hello World"
 
     In [6]: s.upper()
     Out[6]: 'HELLO WORLD'
 
 Another example::
 
     In [8]: 1+3
     Out[8]: 4
 
 Just like in IPython docstrings, you can use all IPython syntax and features::
 
-    In [9]: !echo "hello"
+    In [9]: !echo hello
     hello
 
     In [10]: a='hi'
 
     In [11]: !echo $a
     hi

IPython/testing/tests/test_decorators.py

@@ -1,164 +1,168 b''
 """Tests for the decorators we've created for IPython.
 """
 
 # Module imports
 # Std lib
 import inspect
 import sys
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.testing import decorators as dec
+from IPython.testing.skipdoctest import skip_doctest
 
 #-----------------------------------------------------------------------------
 # Utilities
 
 # Note: copied from OInspect, kept here so the testing stuff doesn't create
 # circular dependencies and is easier to reuse.
 def getargspec(obj):
     """Get the names and default values of a function's arguments.
 
     A tuple of four things is returned: (args, varargs, varkw, defaults).
     'args' is a list of the argument names (it may contain nested lists).
     'varargs' and 'varkw' are the names of the * and ** arguments or None.
     'defaults' is an n-tuple of the default values of the last n arguments.
 
     Modified version of inspect.getargspec from the Python Standard
     Library."""
 
     if inspect.isfunction(obj):
         func_obj = obj
     elif inspect.ismethod(obj):
         func_obj = obj.__func__
     else:
         raise TypeError('arg is not a Python function')
     args, varargs, varkw = inspect.getargs(func_obj.__code__)
     return args, varargs, varkw, func_obj.__defaults__
 
 #-----------------------------------------------------------------------------
 # Testing functions
 
 @dec.as_unittest
 def trivial():
     """A trivial test"""
     pass
 
 
 @dec.skip()
 def test_deliberately_broken():
     """A deliberately broken test - we want to skip this one."""
     1/0
 
 @dec.skip('Testing the skip decorator')
 def test_deliberately_broken2():
     """Another deliberately broken test - we want to skip this one."""
     1/0
 
 
 # Verify that we can correctly skip the doctest for a function at will, but
 # that the docstring itself is NOT destroyed by the decorator.
+@skip_doctest
 def doctest_bad(x,y=1,**k):
     """A function whose doctest we need to skip.
 
     >>> 1+1
     3
     """
     print('x:',x)
     print('y:',y)
     print('k:',k)
 
 
 def call_doctest_bad():
     """Check that we can still call the decorated functions.
     
     >>> doctest_bad(3,y=4)
     x: 3
     y: 4
     k: {}
     """
     pass
 
 
 def test_skip_dt_decorator():
     """Doctest-skipping decorator should preserve the docstring.
     """
     # Careful: 'check' must be a *verbatim* copy of the doctest_bad docstring!
     check = """A function whose doctest we need to skip.
 
     >>> 1+1
     3
     """
     # Fetch the docstring from doctest_bad after decoration.
     val = doctest_bad.__doc__
     
     nt.assert_equal(check,val,"doctest_bad docstrings don't match")
 
 
 # Doctest skipping should work for class methods too
 class FooClass(object):
     """FooClass
 
     Example:
 
     >>> 1+1
     2
     """
 
+    @skip_doctest
     def __init__(self,x):
         """Make a FooClass.
 
         Example:
 
         >>> f = FooClass(3)
         junk
         """
         print('Making a FooClass.')
         self.x = x
         
+    @skip_doctest
     def bar(self,y):
         """Example:
 
         >>> ff = FooClass(3)
         >>> ff.bar(0)
         boom!
         >>> 1/0
         bam!
         """
         return 1/y
 
     def baz(self,y):
         """Example:
 
         >>> ff2 = FooClass(3)
         Making a FooClass.
         >>> ff2.baz(3)
         True
         """
         return self.x==y
 
 
 def test_skip_dt_decorator2():
     """Doctest-skipping decorator should preserve function signature.
     """
     # Hardcoded correct answer
     dtargs = (['x', 'y'], None, 'k', (1,))
     # Introspect out the value
     dtargsr = getargspec(doctest_bad)
     assert dtargsr==dtargs, \
            "Incorrectly reconstructed args for doctest_bad: %s" % (dtargsr,)
 
 
 @dec.skip_linux
 def test_linux():
     nt.assert_false(sys.platform.startswith('linux'),"This test can't run under linux")
 
 @dec.skip_win32
 def test_win32():
     nt.assert_not_equal(sys.platform,'win32',"This test can't run under windows")
 
 @dec.skip_osx
 def test_osx():
     nt.assert_not_equal(sys.platform,'darwin',"This test can't run under osx")
 

IPython/utils/path.py

@@ -1,440 +1,440 b''
 # encoding: utf-8
 """
 Utilities for path handling.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 import errno
 import shutil
 import random
 import glob
 from warnings import warn
 
 from IPython.utils.process import system
 from IPython.utils.decorators import undoc
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 fs_encoding = sys.getfilesystemencoding()
 
 def _writable_dir(path):
     """Whether `path` is a directory, to which the user has write access."""
     return os.path.isdir(path) and os.access(path, os.W_OK)
 
 if sys.platform == 'win32':
     def _get_long_path_name(path):
         """Get a long path name (expand ~) on Windows using ctypes.
 
         Examples
         --------
 
-        >>> get_long_path_name('c:\\docume~1')
+        >>> get_long_path_name('c:\\\\docume~1')
         'c:\\\\Documents and Settings'
 
         """
         try:
             import ctypes
         except ImportError as e: 
             raise ImportError('you need to have ctypes installed for this to work') from e
         _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
         _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
             ctypes.c_uint ]
 
         buf = ctypes.create_unicode_buffer(260)
         rv = _GetLongPathName(path, buf, 260)
         if rv == 0 or rv > 260:
             return path
         else:
             return buf.value
 else:
     def _get_long_path_name(path):
         """Dummy no-op."""
         return path
 
 
 
 def get_long_path_name(path):
     """Expand a path into its long form.
 
     On Windows this expands any ~ in the paths. On other platforms, it is
     a null operation.
     """
     return _get_long_path_name(path)
 
 
 def unquote_filename(name, win32=(sys.platform=='win32')):
     """ On Windows, remove leading and trailing quotes from filenames.
 
     This function has been deprecated and should not be used any more:
     unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
     """
     warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
          "be used anymore", DeprecationWarning, stacklevel=2)
     if win32:
         if name.startswith(("'", '"')) and name.endswith(("'", '"')):
             name = name[1:-1]
     return name
 
 
 def compress_user(path):
     """Reverse of :func:`os.path.expanduser`
     """
     home = os.path.expanduser('~')
     if path.startswith(home):
         path =  "~" + path[len(home):]
     return path
 
 def get_py_filename(name, force_win32=None):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found.
     """
 
     name = os.path.expanduser(name)
     if force_win32 is not None:
         warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
              "since IPython 5.0 and should not be used anymore",
             DeprecationWarning, stacklevel=2)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError('File `%r` not found.' % name)
 
 
 def filefind(filename: str, path_dirs=None) -> str:
     """Find a file by looking through a sequence of paths.
 
     This iterates through a sequence of paths looking for a file and returns
     the full, absolute path of the first occurrence of the file.  If no set of
     path dirs is given, the filename is tested as is, after running through
     :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
 
         filefind('myfile.txt')
 
     will find the file in the current working dir, but::
 
         filefind('~/myfile.txt')
 
     Will find the file in the users home directory.  This function does not
     automatically try any paths, such as the cwd or the user's home directory.
 
     Parameters
     ----------
     filename : str
         The filename to look for.
     path_dirs : str, None or sequence of str
         The sequence of paths to look for the file in.  If None, the filename
         need to be absolute or be in the cwd.  If a string, the string is
         put into a sequence and the searched.  If a sequence, walk through
         each element and join with ``filename``, calling :func:`expandvars`
         and :func:`expanduser` before testing for existence.
 
     Returns
     -------
     path : str
         returns absolute path to file.
 
     Raises
     ------
     IOError
     """
 
     # If paths are quoted, abspath gets confused, strip them...
     filename = filename.strip('"').strip("'")
     # If the input is an absolute path, just check it exists
     if os.path.isabs(filename) and os.path.isfile(filename):
         return filename
 
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, str):
         path_dirs = (path_dirs,)
 
     for path in path_dirs:
         if path == '.': path = os.getcwd()
         testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
 
     raise IOError("File %r does not exist in any of the search paths: %r" %
                   (filename, path_dirs) )
 
 
 class HomeDirError(Exception):
     pass
 
 
 def get_home_dir(require_writable=False) -> str:
     """Return the 'home' directory, as a unicode string.
 
     Uses os.path.expanduser('~'), and checks for writability.
 
     See stdlib docs for how this is determined.
     For Python <3.8, $HOME is first priority on *ALL* platforms.
     For Python >=3.8 on Windows, %HOME% is no longer considered.
 
     Parameters
     ----------
     require_writable : bool [default: False]
         if True:
             guarantees the return value is a writable directory, otherwise
             raises HomeDirError
         if False:
             The path is resolved, but it is not guaranteed to exist or be writable.
     """
 
     homedir = os.path.expanduser('~')
     # Next line will make things work even when /home/ is a symlink to
     # /usr/home as it is on FreeBSD, for example
     homedir = os.path.realpath(homedir)
 
     if not _writable_dir(homedir) and os.name == 'nt':
         # expanduser failed, use the registry to get the 'My Documents' folder.
         try:
             import winreg as wreg
             with wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
                 r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
             ) as key:
                 homedir = wreg.QueryValueEx(key,'Personal')[0]
         except:
             pass
 
     if (not require_writable) or _writable_dir(homedir):
         assert isinstance(homedir, str), "Homedir should be unicode not bytes"
         return homedir
     else:
         raise HomeDirError('%s is not a writable dir, '
                 'set $HOME environment variable to override' % homedir)
 
 def get_xdg_dir():
     """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.config if empty OR not set
         xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
         if xdg and _writable_dir(xdg):
             assert isinstance(xdg, str)
             return xdg
 
     return None
 
 
 def get_xdg_cache_dir():
     """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.cache if empty OR not set
         xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
         if xdg and _writable_dir(xdg):
             assert isinstance(xdg, str)
             return xdg
 
     return None
 
 
 @undoc
 def get_ipython_dir():
     warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_dir
     return get_ipython_dir()
 
 @undoc
 def get_ipython_cache_dir():
     warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_cache_dir
     return get_ipython_cache_dir()
 
 @undoc
 def get_ipython_package_dir():
     warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_package_dir
     return get_ipython_package_dir()
 
 @undoc
 def get_ipython_module_path(module_str):
     warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_module_path
     return get_ipython_module_path(module_str)
 
 @undoc
 def locate_profile(profile='default'):
     warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import locate_profile
     return locate_profile(profile=profile)
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
 
        In [2]: os.environ['FOO']='test'
 
        In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
     # This is a pretty subtle hack. When expand user is given a UNC path
     # on Windows (\\server\share$\%username%), os.path.expandvars, removes
     # the $ to get (\\server\share\%username%). I think it considered $
     # alone an empty var. But, we need the $ to remains there (it indicates
     # a hidden share).
     if os.name=='nt':
         s = s.replace('$\\', 'IPYTHON_TEMP')
     s = os.path.expandvars(os.path.expanduser(s))
     if os.name=='nt':
         s = s.replace('IPYTHON_TEMP', '$\\')
     return s
 
 
 def unescape_glob(string):
     """Unescape glob pattern in `string`."""
     def unescape(s):
         for pattern in '*[]!?':
             s = s.replace(r'\{0}'.format(pattern), pattern)
         return s
     return '\\'.join(map(unescape, string.split('\\\\')))
 
 
 def shellglob(args):
     """
     Do glob expansion for each element in `args` and return a flattened list.
 
     Unmatched glob pattern will remain as-is in the returned list.
 
     """
     expanded = []
     # Do not unescape backslash in Windows as it is interpreted as
     # path separator:
     unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
     for a in args:
         expanded.extend(glob.glob(a) or [unescape(a)])
     return expanded
 
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         system(cmd)
 
 
 ENOLINK = 1998
 
 def link(src, dst):
     """Hard links ``src`` to ``dst``, returning 0 or errno.
 
     Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
     supported by the operating system.
     """
 
     if not hasattr(os, "link"):
         return ENOLINK
     link_errno = 0
     try:
         os.link(src, dst)
     except OSError as e:
         link_errno = e.errno
     return link_errno
 
 
 def link_or_copy(src, dst):
     """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
 
     Attempts to maintain the semantics of ``shutil.copy``.
 
     Because ``os.link`` does not overwrite files, a unique temporary file
     will be used if the target already exists, then that file will be moved
     into place.
     """
 
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
 
     link_errno = link(src, dst)
     if link_errno == errno.EEXIST:
         if os.stat(src).st_ino == os.stat(dst).st_ino:
             # dst is already a hard link to the correct file, so we don't need
             # to do anything else. If we try to link and rename the file
             # anyway, we get duplicate files - see http://bugs.python.org/issue21876
             return
 
         new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
         try:
             link_or_copy(src, new_dst)
         except:
             try:
                 os.remove(new_dst)
             except OSError:
                 pass
             raise
         os.rename(new_dst, dst)
     elif link_errno != 0:
         # Either link isn't supported, or the filesystem doesn't support
         # linking, or 'src' and 'dst' are on different filesystems.
         shutil.copy(src, dst)
 
 def ensure_dir_exists(path, mode=0o755):
     """ensure that a directory exists
 
     If it doesn't exist, try to create it and protect against a race condition
     if another process is doing the same.
 
     The default permissions are 755, which differ from os.makedirs default of 777.
     """
     if not os.path.exists(path):
         try:
             os.makedirs(path, mode=mode)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
     elif not os.path.isdir(path):
         raise IOError("%r exists but is not a directory" % path)