Merge pull request #11250 from Carreau/invalid-escape-sequence

Use raw string when invalid escape sequence present.

IPython/core/completer.py

@@ -1,2060 +1,2060 b''
 """Completion for IPython.
 
 This module started as fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3,
 
 This module now support a wide variety of completion mechanism both available
 for normal classic Python code, as well as completer for IPython specific
 Syntax like magics.
 
 Latex and Unicode completion
 ============================
 
 IPython and compatible frontends not only can complete your code, but can help
 you to input a wide range of characters. In particular we allow you to insert
 a unicode character using the tab completion mechanism.
 
 Forward latex/unicode completion
 --------------------------------
 
 Forward completion allows you to easily type a unicode character using its latex
 name, or unicode long description. To do so type a backslash follow by the
 relevant name and press tab:
 
 
 Using latex completion:
 
 .. code::
 
     \\alpha<tab>
     α
 
 or using unicode completion:
 
 
 .. code::
 
     \\greek small letter alpha<tab>
     α
 
 
 Only valid Python identifiers will complete. Combining characters (like arrow or
 dots) are also available, unlike latex they need to be put after the their
 counterpart that is to say, `F\\\\vec<tab>` is correct, not `\\\\vec<tab>F`.
 
 Some browsers are known to display combining characters incorrectly.
 
 Backward latex completion
 -------------------------
 
 It is sometime challenging to know how to type a character, if you are using
 IPython, or any compatible frontend you can prepend backslash to the character
 and press `<tab>` to expand it to its latex form.
 
 .. code::
 
     \\α<tab>
     \\alpha
 
 
 Both forward and backward completions can be deactivated by setting the
 ``Completer.backslash_combining_completions`` option to ``False``.
 
 
 Experimental
 ============
 
 Starting with IPython 6.0, this module can make use of the Jedi library to
 generate completions both using static analysis of the code, and dynamically
 inspecting multiple namespaces. The APIs attached to this new mechanism is
 unstable and will raise unless use in an :any:`provisionalcompleter` context
 manager.
 
 You will find that the following are experimental:
 
     - :any:`provisionalcompleter`
     - :any:`IPCompleter.completions`
     - :any:`Completion`
     - :any:`rectify_completions`
 
 .. note::
 
     better name for :any:`rectify_completions` ?
 
 We welcome any feedback on these new API, and we also encourage you to try this
 module in debug mode (start IPython with ``--Completer.debug=True``) in order
 to have extra logging information is :any:`jedi` is crashing, or if current
 IPython completer pending deprecations are returning results not yet handled
 by :any:`jedi`
 
 Using Jedi for tab completion allow snippets like the following to work without
 having to execute any code:
 
    >>> myvar = ['hello', 42]
    ... myvar[1].bi<tab>
 
 Tab completion will be able to infer that ``myvar[1]`` is a real number without
 executing any code unlike the previously available ``IPCompleter.greedy``
 option.
 
 Be sure to update :any:`jedi` to the latest stable version or to try the
 current development version to get better completions.
 """
 
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 #
 # Some of this code originated from rlcompleter in the Python standard library
 # Copyright (C) 2001 Python Software Foundation, www.python.org
 
 
 import __main__
 import builtins as builtin_mod
 import glob
 import time
 import inspect
 import itertools
 import keyword
 import os
 import re
 import sys
 import unicodedata
 import string
 import warnings
 
 from contextlib import contextmanager
 from importlib import import_module
 from typing import Iterator, List, Tuple, Iterable, Union
 from types import SimpleNamespace
 
 from traitlets.config.configurable import Configurable
 from IPython.core.error import TryNext
 from IPython.core.inputsplitter import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
 from IPython.core.oinspect import InspectColors
 from IPython.utils import generics
 from IPython.utils.dir2 import dir2, get_real_method
 from IPython.utils.process import arg_split
 from traitlets import Bool, Enum, observe, Int
 
 # skip module docstests
 skip_doctest = True
 
 try:
     import jedi
     jedi.settings.case_insensitive_completion = False
     import jedi.api.helpers
     import jedi.api.classes
     JEDI_INSTALLED = True
 except ImportError:
     JEDI_INSTALLED = False
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Public API
 __all__ = ['Completer','IPCompleter']
 
 if sys.platform == 'win32':
     PROTECTABLES = ' '
 else:
     PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
 
 # Protect against returning an enormous number of completions which the frontend
 # may have trouble processing.
 MATCHES_LIMIT = 500
 
 _deprecation_readline_sentinel = object()
 
 
 class ProvisionalCompleterWarning(FutureWarning):
     """
     Exception raise by an experimental feature in this module.
 
     Wrap code in :any:`provisionalcompleter` context manager if you
     are certain you want to use an unstable feature.
     """
     pass
 
 warnings.filterwarnings('error', category=ProvisionalCompleterWarning)
 
 @contextmanager
 def provisionalcompleter(action='ignore'):
     """
 
 
     This contest manager has to be used in any place where unstable completer
     behavior and API may be called.
 
     >>> with provisionalcompleter():
     ...     completer.do_experimetal_things() # works
 
     >>> completer.do_experimental_things() # raises.
 
     .. note:: Unstable
 
         By using this context manager you agree that the API in use may change
         without warning, and that you won't complain if they do so.
 
         You also understand that if the API is not to you liking you should report
         a bug to explain your use case upstream and improve the API and will loose
         credibility if you complain after the API is make stable.
 
         We'll be happy to get your feedback , feature request and improvement on
         any of the unstable APIs !
     """
     with warnings.catch_warnings():
         warnings.filterwarnings(action, category=ProvisionalCompleterWarning)
         yield
 
 
 def has_open_quotes(s):
     """Return whether a string has open quotes.
 
     This simply counts whether the number of quote characters of either type in
     the string is odd.
 
     Returns
     -------
     If there is an open quote, the quote character is returned.  Else, return
     False.
     """
     # We check " first, then ', so complex cases with nested quotes will get
     # the " to take precedence.
     if s.count('"') % 2:
         return '"'
     elif s.count("'") % 2:
         return "'"
     else:
         return False
 
 
 def protect_filename(s, protectables=PROTECTABLES):
     """Escape a string to protect certain characters."""
     if set(s) & set(protectables):
         if sys.platform == "win32":
             return '"' + s + '"'
         else:
             return "".join(("\\" + c if c in protectables else c) for c in s)
     else:
         return s
 
 
 def expand_user(path:str) -> Tuple[str, bool, str]:
     """Expand ``~``-style usernames in strings.
 
     This is similar to :func:`os.path.expanduser`, but it computes and returns
     extra information that will be useful if the input was being used in
     computing completions, and you wish to return the completions with the
     original '~' instead of its expanded value.
 
     Parameters
     ----------
     path : str
       String to be expanded.  If no ~ is present, the output is the same as the
       input.
 
     Returns
     -------
     newpath : str
       Result of ~ expansion in the input path.
     tilde_expand : bool
       Whether any expansion was performed or not.
     tilde_val : str
       The value that ~ was replaced with.
     """
     # Default values
     tilde_expand = False
     tilde_val = ''
     newpath = path
 
     if path.startswith('~'):
         tilde_expand = True
         rest = len(path)-1
         newpath = os.path.expanduser(path)
         if rest:
             tilde_val = newpath[:-rest]
         else:
             tilde_val = newpath
 
     return newpath, tilde_expand, tilde_val
 
 
 def compress_user(path:str, tilde_expand:bool, tilde_val:str) -> str:
     """Does the opposite of expand_user, with its outputs.
     """
     if tilde_expand:
         return path.replace(tilde_val, '~')
     else:
         return path
 
 
 def completions_sorting_key(word):
     """key for sorting completions
 
     This does several things:
 
     - Demote any completions starting with underscores to the end
     - Insert any %magic and %%cellmagic completions in the alphabetical order
       by their name
     """
     prio1, prio2 = 0, 0
 
     if word.startswith('__'):
         prio1 = 2
     elif word.startswith('_'):
         prio1 = 1
 
     if word.endswith('='):
         prio1 = -1
 
     if word.startswith('%%'):
         # If there's another % in there, this is something else, so leave it alone
         if not "%" in word[2:]:
             word = word[2:]
             prio2 = 2
     elif word.startswith('%'):
         if not "%" in word[1:]:
             word = word[1:]
             prio2 = 1
 
     return prio1, word, prio2
 
 
 class _FakeJediCompletion:
     """
     This is a workaround to communicate to the UI that Jedi has crashed and to
     report a bug. Will be used only id :any:`IPCompleter.debug` is set to true.
 
     Added in IPython 6.0 so should likely be removed for 7.0
 
     """
 
     def __init__(self, name):
 
         self.name = name
         self.complete = name
         self.type = 'crashed'
         self.name_with_symbols = name
         self.signature = ''
         self._origin = 'fake'
 
     def __repr__(self):
         return '<Fake completion object jedi has crashed>'
 
 
 class Completion:
     """
     Completion object used and return by IPython completers.
 
     .. warning:: Unstable
         
         This function is unstable, API may change without warning.
         It will also raise unless use in proper context manager.
 
     This act as a middle ground :any:`Completion` object between the
     :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion
     object. While Jedi need a lot of information about evaluator and how the
     code should be ran/inspected, PromptToolkit (and other frontend) mostly
     need user facing information.
 
     - Which range should be replaced replaced by what.
     - Some metadata (like completion type), or meta information to displayed to
       the use user.
 
     For debugging purpose we can also store the origin of the completion (``jedi``,
     ``IPython.python_matches``, ``IPython.magics_matches``...).
     """
 
     __slots__ = ['start', 'end', 'text', 'type', 'signature', '_origin']
 
     def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin='', signature='') -> None:
         warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
         self.start = start
         self.end = end
         self.text = text
         self.type = type
         self.signature = signature
         self._origin = _origin
 
     def __repr__(self):
         return '<Completion start=%s end=%s text=%r type=%r, signature=%r,>' % \
                 (self.start, self.end, self.text, self.type or '?', self.signature or '?')
 
     def __eq__(self, other)->Bool:
         """
         Equality and hash do not hash the type (as some completer may not be
         able to infer the type), but are use to (partially) de-duplicate
         completion.
 
         Completely de-duplicating completion is a bit tricker that just
         comparing as it depends on surrounding text, which Completions are not
         aware of.
         """
         return self.start == other.start and \
             self.end == other.end and \
             self.text == other.text
 
     def __hash__(self):
         return hash((self.start, self.end, self.text))
 
 
 _IC = Iterable[Completion]
 
 
 def _deduplicate_completions(text: str, completions: _IC)-> _IC:
     """
     Deduplicate a set of completions.
 
     .. warning:: Unstable
 
         This function is unstable, API may change without warning.
 
     Parameters
     ----------
     text: str
         text that should be completed.
     completions: Iterator[Completion]
         iterator over the completions to deduplicate
 
     Yields
     ------
     `Completions` objects
 
 
     Completions coming from multiple sources, may be different but end up having
     the same effect when applied to ``text``. If this is the case, this will
     consider completions as equal and only emit the first encountered.
 
     Not folded in `completions()` yet for debugging purpose, and to detect when
     the IPython completer does return things that Jedi does not, but should be
     at some point.
     """
     completions = list(completions)
     if not completions:
         return
 
     new_start = min(c.start for c in completions)
     new_end = max(c.end for c in completions)
 
     seen = set()
     for c in completions:
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if new_text not in seen:
             yield c
             seen.add(new_text)
 
 
 def rectify_completions(text: str, completions: _IC, *, _debug=False)->_IC:
     """
     Rectify a set of completions to all have the same ``start`` and ``end``
 
     .. warning:: Unstable
 
         This function is unstable, API may change without warning.
         It will also raise unless use in proper context manager.
 
     Parameters
     ----------
     text: str
         text that should be completed.
     completions: Iterator[Completion]
         iterator over the completions to rectify
 
 
     :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though
     the Jupyter Protocol requires them to behave like so. This will readjust
     the completion to have the same ``start`` and ``end`` by padding both
     extremities with surrounding text.
 
     During stabilisation should support a ``_debug`` option to log which
     completion are return by the IPython completer and not found in Jedi in
     order to make upstream bug report.
     """
     warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). "
                  "It may change without warnings. "
                  "Use in corresponding context manager.",
                   category=ProvisionalCompleterWarning, stacklevel=2)
 
     completions = list(completions)
     if not completions:
         return
     starts = (c.start for c in completions)
     ends = (c.end for c in completions)
 
     new_start = min(starts)
     new_end = max(ends)
 
     seen_jedi = set()
     seen_python_matches = set()
     for c in completions:
         new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
         if c._origin == 'jedi':
             seen_jedi.add(new_text)
         elif c._origin == 'IPCompleter.python_matches':
             seen_python_matches.add(new_text)
         yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin, signature=c.signature)
     diff = seen_python_matches.difference(seen_jedi)
     if diff and _debug:
         print('IPython.python matches have extras:', diff)
 
 
 if sys.platform == 'win32':
     DELIMS = ' \t\n`!@#$^&*()=+[{]}|;\'",<>?'
 else:
     DELIMS = ' \t\n`!@#$^&*()=+[{]}\\|;:\'",<>?'
 
 GREEDY_DELIMS = ' =\r\n'
 
 
 class CompletionSplitter(object):
     """An object to split an input line in a manner similar to readline.
 
     By having our own implementation, we can expose readline-like completion in
     a uniform manner to all frontends.  This object only needs to be given the
     line of text to be split and the cursor position on said line, and it
     returns the 'word' to be completed on at the cursor after splitting the
     entire line.
 
     What characters are used as splitting delimiters can be controlled by
     setting the ``delims`` attribute (this is a property that internally
     automatically builds the necessary regular expression)"""
 
     # Private interface
 
     # A string of delimiter characters.  The default value makes sense for
     # IPython's most typical usage patterns.
     _delims = DELIMS
 
     # The expression (a normal string) to be compiled into a regular expression
     # for actual splitting.  We store it as an attribute mostly for ease of
     # debugging, since this type of code can be so tricky to debug.
     _delim_expr = None
 
     # The regular expression that does the actual splitting
     _delim_re = None
 
     def __init__(self, delims=None):
         delims = CompletionSplitter._delims if delims is None else delims
         self.delims = delims
 
     @property
     def delims(self):
         """Return the string of delimiter characters."""
         return self._delims
 
     @delims.setter
     def delims(self, delims):
         """Set the delimiters for line splitting."""
         expr = '[' + ''.join('\\'+ c for c in delims) + ']'
         self._delim_re = re.compile(expr)
         self._delims = delims
         self._delim_expr = expr
 
     def split_line(self, line, cursor_pos=None):
         """Split a line of text with a cursor at the given position.
         """
         l = line if cursor_pos is None else line[:cursor_pos]
         return self._delim_re.split(l)[-1]
 
 
 
 class Completer(Configurable):
 
     greedy = Bool(False,
         help="""Activate greedy completion
         PENDING DEPRECTION. this is now mostly taken care of with Jedi.
 
         This will enable completion on elements of lists, results of function calls, etc.,
         but can be unsafe because the code is actually evaluated on TAB.
         """
     ).tag(config=True)
 
     use_jedi = Bool(default_value=False,
                     help="Experimental: Use Jedi to generate autocompletions. "
                     "Off by default.").tag(config=True)
 
     jedi_compute_type_timeout = Int(default_value=400,
         help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types.
         Set to 0 to stop computing types. Non-zero value lower than 100ms may hurt
         performance by preventing jedi to build its cache.
         """).tag(config=True)
 
     debug = Bool(default_value=False,
                  help='Enable debug for the Completer. Mostly print extra '
                       'information for experimental jedi integration.')\
                       .tag(config=True)
 
     backslash_combining_completions = Bool(True, 
         help="Enable unicode completions, e.g. \\alpha<tab> . "
              "Includes completion of latex commands, unicode names, and expanding "
              "unicode characters back to latex commands.").tag(config=True)
 
 
 
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
         Completer(namespace=ns, global_namespace=ns2) -> completer instance.
 
         If unspecified, the default namespace where completions are performed
         is __main__ (technically, __main__.__dict__). Namespaces should be
         given as dictionaries.
 
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
             self.use_main_ns = True
         else:
             self.use_main_ns = False
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
         if global_namespace is None:
             self.global_namespace = {}
         else:
             self.global_namespace = global_namespace
 
         super(Completer, self).__init__(**kwargs)
 
     def complete(self, text, state):
         """Return the next possible completion for 'text'.
 
         This is called successively with state == 0, 1, 2, ... until it
         returns None.  The completion should begin with 'text'.
 
         """
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         if state == 0:
             if "." in text:
                 self.matches = self.attr_matches(text)
             else:
                 self.matches = self.global_matches(text)
         try:
             return self.matches[state]
         except IndexError:
             return None
 
     def global_matches(self, text):
         """Compute matches when text is a simple name.
 
         Return a list of all keywords, built-in functions and names currently
         defined in self.namespace or self.global_namespace that match.
 
         """
         matches = []
         match_append = matches.append
         n = len(text)
         for lst in [keyword.kwlist,
                     builtin_mod.__dict__.keys(),
                     self.namespace.keys(),
                     self.global_namespace.keys()]:
             for word in lst:
                 if word[:n] == text and word != "__builtins__":
                     match_append(word)
 
         snake_case_re = re.compile(r"[^_]+(_[^_]+)+?\Z")
         for lst in [self.namespace.keys(),
                     self.global_namespace.keys()]:
             shortened = {"_".join([sub[0] for sub in word.split('_')]) : word
                          for word in lst if snake_case_re.match(word)}
             for word in shortened.keys():
                 if word[:n] == text and word != "__builtins__":
                     match_append(shortened[word])
         return matches
 
     def attr_matches(self, text):
         """Compute matches when text contains a dot.
 
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
         possible completions.  (For class instances, class members are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
         with a __getattr__ hook is evaluated.
 
         """
 
         # Another option, seems to work great. Catches things like ''.<tab>
         m = re.match(r"(\S+(\.\w+)*)\.(\w*)$", text)
     
         if m:
             expr, attr = m.group(1, 3)
         elif self.greedy:
             m2 = re.match(r"(.+)\.(\w*)$", self.line_buffer)
             if not m2:
                 return []
             expr, attr = m2.group(1,2)
         else:
             return []
     
         try:
             obj = eval(expr, self.namespace)
         except:
             try:
                 obj = eval(expr, self.global_namespace)
             except:
                 return []
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
         else:
             words = dir2(obj)
 
         try:
             words = generics.complete_object(obj, words)
         except TryNext:
             pass
         except AssertionError:
             raise
         except Exception:
             # Silence errors from completion function
             #raise # dbg
             pass
         # Build match list to return
         n = len(attr)
         return [u"%s.%s" % (expr, w) for w in words if w[:n] == attr ]
 
 
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
         words = getattr(obj, '__all__')
     except:
         return []
  
     return [w for w in words if isinstance(w, str)]
 
 
 def match_dict_keys(keys: List[str], prefix: str, delims: str):
     """Used by dict_key_matches, matching the prefix to a list of keys
 
     Parameters
     ==========
     keys:
         list of keys in dictionary currently being completed.
     prefix:
         Part of the text already typed by the user. e.g. `mydict[b'fo`
     delims:
         String of delimiters to consider when finding the current key.
 
     Returns
     =======
 
     A tuple of three elements: ``quote``, ``token_start``, ``matched``, with
     ``quote`` being the quote that need to be used to close current string.
     ``token_start`` the position where the replacement should start occurring,
     ``matches`` a list of replacement/completion
 
     """
     if not prefix:
         return None, 0, [repr(k) for k in keys
                       if isinstance(k, (str, bytes))]
     quote_match = re.search('["\']', prefix)
     quote = quote_match.group()
     try:
         prefix_str = eval(prefix + quote, {})
     except Exception:
         return None, 0, []
 
     pattern = '[^' + ''.join('\\' + c for c in delims) + ']*$'
     token_match = re.search(pattern, prefix, re.UNICODE)
     token_start = token_match.start()
     token_prefix = token_match.group()
 
     matched = []
     for key in keys:
         try:
             if not key.startswith(prefix_str):
                 continue
         except (AttributeError, TypeError, UnicodeError):
             # Python 3+ TypeError on b'a'.startswith('a') or vice-versa
             continue
 
         # reformat remainder of key to begin with prefix
         rem = key[len(prefix_str):]
         # force repr wrapped in '
         rem_repr = repr(rem + '"') if isinstance(rem, str) else repr(rem + b'"')
         if rem_repr.startswith('u') and prefix[0] not in 'uU':
             # Found key is unicode, but prefix is Py2 string.
             # Therefore attempt to interpret key as string.
             try:
                 rem_repr = repr(rem.encode('ascii') + '"')
             except UnicodeEncodeError:
                 continue
 
         rem_repr = rem_repr[1 + rem_repr.index("'"):-2]
         if quote == '"':
             # The entered prefix is quoted with ",
             # but the match is quoted with '.
             # A contained " hence needs escaping for comparison:
             rem_repr = rem_repr.replace('"', '\\"')
 
         # then reinsert prefix from start of token
         matched.append('%s%s' % (token_prefix, rem_repr))
     return quote, token_start, matched
 
 
 def cursor_to_position(text:str, line:int, column:int)->int:
     """
 
     Convert the (line,column) position of the cursor in text to an offset in a
     string.
 
     Parameters
     ----------
 
     text : str
         The text in which to calculate the cursor offset
     line : int
         Line of the cursor; 0-indexed
     column : int
         Column of the cursor 0-indexed
 
     Return
     ------
         Position of the cursor in ``text``, 0-indexed.
 
     See Also
     --------
     position_to_cursor: reciprocal of this function
 
     """
     lines = text.split('\n')
     assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
 
     return sum(len(l) + 1 for l in lines[:line]) + column
 
 def position_to_cursor(text:str, offset:int)->Tuple[int, int]:
     """
     Convert the position of the cursor in text (0 indexed) to a line
     number(0-indexed) and a column number (0-indexed) pair
 
     Position should be a valid position in ``text``.
 
     Parameters
     ----------
 
     text : str
         The text in which to calculate the cursor offset
     offset : int
         Position of the cursor in ``text``, 0-indexed.
 
     Return
     ------
     (line, column) : (int, int)
         Line of the cursor; 0-indexed, column of the cursor 0-indexed
 
 
     See Also
     --------
     cursor_to_position : reciprocal of this function
 
 
     """
 
     assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text))
 
     before = text[:offset]
     blines = before.split('\n')  # ! splitnes trim trailing \n
     line = before.count('\n')
     col = len(blines[-1])
     return line, col
 
 
 def _safe_isinstance(obj, module, class_name):
     """Checks if obj is an instance of module.class_name if loaded
     """
     return (module in sys.modules and
             isinstance(obj, getattr(import_module(module), class_name)))
 
 
 def back_unicode_name_matches(text):
     u"""Match unicode characters back to unicode name
     
     This does  ``☃`` -> ``\\snowman``
 
     Note that snowman is not a valid python3 combining character but will be expanded.
     Though it will not recombine back to the snowman character by the completion machinery.
 
     This will not either back-complete standard sequences like \\n, \\b ...
     
     Used on Python 3 only.
     """
     if len(text)<2:
         return u'', ()
     maybe_slash = text[-2]
     if maybe_slash != '\\':
         return u'', ()
 
     char = text[-1]
     # no expand on quote for completion in strings.
     # nor backcomplete standard ascii keys
     if char in string.ascii_letters or char in ['"',"'"]:
         return u'', ()
     try :
         unic = unicodedata.name(char)
         return '\\'+char,['\\'+unic]
     except KeyError:
         pass
     return u'', ()
 
 def back_latex_name_matches(text:str):
     """Match latex characters back to unicode name
     
     This does ``\\ℵ`` -> ``\\aleph``
 
     Used on Python 3 only.
     """
     if len(text)<2:
         return u'', ()
     maybe_slash = text[-2]
     if maybe_slash != '\\':
         return u'', ()
 
 
     char = text[-1]
     # no expand on quote for completion in strings.
     # nor backcomplete standard ascii keys
     if char in string.ascii_letters or char in ['"',"'"]:
         return u'', ()
     try :
         latex = reverse_latex_symbol[char]
         # '\\' replace the \ as well
         return '\\'+char,[latex]
     except KeyError:
         pass
     return u'', ()
 
 
 def _formatparamchildren(parameter) -> str:
     """
     Get parameter name and value from Jedi Private API
 
     Jedi does not expose a simple way to get `param=value` from its API.
 
     Parameter
     =========
 
     parameter:
         Jedi's function `Param`
 
     Returns
     =======
 
     A string like 'a', 'b=1', '*args', '**kwargs'
 
 
     """
     description = parameter.description
     if not description.startswith('param '):
         raise ValueError('Jedi function parameter description have change format.'
                          'Expected "param ...", found %r".' % description)
     return description[6:]
 
 def _make_signature(completion)-> str:
     """
     Make the signature from a jedi completion
 
     Parameter
     =========
 
     completion: jedi.Completion
         object does not complete a function type
 
     Returns
     =======
 
     a string consisting of the function signature, with the parenthesis but
     without the function name. example:
     `(a, *args, b=1, **kwargs)`
 
     """
 
     return '(%s)'% ', '.join([f for f in (_formatparamchildren(p) for p in completion.params) if f])
 
 class IPCompleter(Completer):
     """Extension of the completer class with IPython-specific features"""
     
     @observe('greedy')
     def _greedy_changed(self, change):
         """update the splitter and readline delims when greedy is changed"""
         if change['new']:
             self.splitter.delims = GREEDY_DELIMS
         else:
             self.splitter.delims = DELIMS
     
     merge_completions = Bool(True,
         help="""Whether to merge completion results into a single list
         
         If False, only the completion results from the first non-empty
         completer will be returned.
         """
     ).tag(config=True)
     omit__names = Enum((0,1,2), default_value=2,
         help="""Instruct the completer to omit private method names
         
         Specifically, when completing on ``object.<tab>``.
         
         When 2 [default]: all names that start with '_' will be excluded.
         
         When 1: all 'magic' names (``__foo__``) will be excluded.
         
         When 0: nothing will be excluded.
         """
     ).tag(config=True)
     limit_to__all__ = Bool(False,
         help="""
         DEPRECATED as of version 5.0.
         
         Instruct the completer to use __all__ for the completion
         
         Specifically, when completing on ``object.<tab>``.
         
         When True: only those names in obj.__all__ will be included.
         
         When False [default]: the __all__ attribute is ignored
         """,
     ).tag(config=True)
 
     @observe('limit_to__all__')
     def _limit_to_all_changed(self, change):
         warnings.warn('`IPython.core.IPCompleter.limit_to__all__` configuration '
             'value has been deprecated since IPython 5.0, will be made to have '
             'no effects and then removed in future version of IPython.',
             UserWarning)
 
     def __init__(self, shell=None, namespace=None, global_namespace=None,
                  use_readline=_deprecation_readline_sentinel, config=None, **kwargs):
         """IPCompleter() -> completer
 
         Return a completer object.
 
         Parameters
         ----------
 
         shell
             a pointer to the ipython shell itself.  This is needed
             because this completer knows about magic functions, and those can
             only be accessed via the ipython instance.
 
         namespace : dict, optional
             an optional dict where completions are performed.
 
         global_namespace : dict, optional
             secondary optional dict for completions, to
             handle cases (such as IPython embedded inside functions) where
             both Python scopes are visible.
        
         use_readline : bool, optional
             DEPRECATED, ignored since IPython 6.0, will have no effects
         """
 
         self.magic_escape = ESC_MAGIC
         self.splitter = CompletionSplitter()
 
         if use_readline is not _deprecation_readline_sentinel:
             warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.',
                           DeprecationWarning, stacklevel=2)
 
         # _greedy_changed() depends on splitter and readline being defined:
         Completer.__init__(self, namespace=namespace, global_namespace=global_namespace,
                             config=config, **kwargs)
 
         # List where completion matches will be stored
         self.matches = []
         self.shell = shell
         # Regexp to split filenames with spaces in them
         self.space_name_re = re.compile(r'([^\\] )')
         # Hold a local ref. to glob.glob for speed
         self.glob = glob.glob
 
         # Determine if we are running on 'dumb' terminals, like (X)Emacs
         # buffers, to avoid completion problems.
         term = os.environ.get('TERM','xterm')
         self.dumb_terminal = term in ['dumb','emacs']
 
         # Special handling of backslashes needed in win32 platforms
         if sys.platform == "win32":
             self.clean_glob = self._clean_glob_win32
         else:
             self.clean_glob = self._clean_glob
 
         #regexp to parse docstring for function signature
         self.docstring_sig_re = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         self.docstring_kwd_re = re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
         #use this if positional argument name is also needed
         #= re.compile(r'[\s|\[]*(\w+)(?:\s*=?\s*.*)')
 
         self.magic_arg_matchers = [
             self.magic_config_matches,
             self.magic_color_matches,
         ]
 
         # This is set externally by InteractiveShell
         self.custom_completers = None
 
     @property
     def matchers(self):
         """All active matcher routines for completion"""
         if self.use_jedi:
             return [
                 self.file_matches,
                 self.magic_matches,
                 self.dict_key_matches,
             ]
         else:
             return [
                 self.python_matches,
                 self.file_matches,
                 self.magic_matches,
                 self.python_func_kw_matches,
                 self.dict_key_matches,
             ]
 
     def all_completions(self, text):
         """
         Wrapper around the complete method for the benefit of emacs.
         """
         return self.complete(text)[1]
 
     def _clean_glob(self, text):
         return self.glob("%s*" % text)
 
     def _clean_glob_win32(self,text):
         return [f.replace("\\","/")
                 for f in self.glob("%s*" % text)]
 
     def file_matches(self, text):
         """Match filenames, expanding ~USER type strings.
 
         Most of the seemingly convoluted logic in this completer is an
         attempt to handle filenames with spaces in them.  And yet it's not
         quite perfect, because Python's readline doesn't expose all of the
         GNU readline details needed for this to be done correctly.
 
         For a filename with a space in it, the printed completions will be
         only the parts after what's already been typed (instead of the
         full completions, as is normally done).  I don't think with the
         current (as of Python 2.3) Python readline it's possible to do
         better."""
 
         # chars that require escaping with backslash - i.e. chars
         # that readline treats incorrectly as delimiters, but we
         # don't want to treat as delimiters in filename matching
         # when escaped with backslash
         if text.startswith('!'):
             text = text[1:]
             text_prefix = u'!'
         else:
             text_prefix = u''
 
         text_until_cursor = self.text_until_cursor
         # track strings with open quotes
         open_quotes = has_open_quotes(text_until_cursor)
 
         if '(' in text_until_cursor or '[' in text_until_cursor:
             lsplit = text
         else:
             try:
                 # arg_split ~ shlex.split, but with unicode bugs fixed by us
                 lsplit = arg_split(text_until_cursor)[-1]
             except ValueError:
                 # typically an unmatched ", or backslash without escaped char.
                 if open_quotes:
                     lsplit = text_until_cursor.split(open_quotes)[-1]
                 else:
                     return []
             except IndexError:
                 # tab pressed on empty line
                 lsplit = ""
 
         if not open_quotes and lsplit != protect_filename(lsplit):
             # if protectables are found, do matching on the whole escaped name
             has_protectables = True
             text0,text = text,lsplit
         else:
             has_protectables = False
             text = os.path.expanduser(text)
 
         if text == "":
             return [text_prefix + protect_filename(f) for f in self.glob("*")]
 
         # Compute the matches from the filesystem
         if sys.platform == 'win32':
             m0 = self.clean_glob(text)
         else:
             m0 = self.clean_glob(text.replace('\\', ''))
 
         if has_protectables:
             # If we had protectables, we need to revert our changes to the
             # beginning of filename so that we don't double-write the part
             # of the filename we have so far
             len_lsplit = len(lsplit)
             matches = [text_prefix + text0 +
                        protect_filename(f[len_lsplit:]) for f in m0]
         else:
             if open_quotes:
                 # if we have a string with an open quote, we don't need to
                 # protect the names beyond the quote (and we _shouldn't_, as
                 # it would cause bugs when the filesystem call is made).
                 matches = m0 if sys.platform == "win32" else\
                     [protect_filename(f, open_quotes) for f in m0]
             else:
                 matches = [text_prefix +
                            protect_filename(f) for f in m0]
 
         # Mark directories in input list by appending '/' to their names.
         return [x+'/' if os.path.isdir(x) else x for x in matches]
 
     def magic_matches(self, text):
         """Match magics"""
         # Get all shell magics now rather than statically, so magics loaded at
         # runtime show up too.
         lsm = self.shell.magics_manager.lsmagic()
         line_magics = lsm['line']
         cell_magics = lsm['cell']
         pre = self.magic_escape
         pre2 = pre+pre
 
         explicit_magic = text.startswith(pre)
 
         # Completion logic:
         # - user gives %%: only do cell magics
         # - user gives %: do both line and cell magics
         # - no prefix: do both
         # In other words, line magics are skipped if the user gives %% explicitly
         #
         # We also exclude magics that match any currently visible names:
         # https://github.com/ipython/ipython/issues/4877, unless the user has
         # typed a %:
         # https://github.com/ipython/ipython/issues/10754
         bare_text = text.lstrip(pre)
         global_matches = self.global_matches(bare_text)
         if not explicit_magic:
             def matches(magic):
                 """
                 Filter magics, in particular remove magics that match
                 a name present in global namespace.
                 """
                 return ( magic.startswith(bare_text) and
                          magic not in global_matches )
         else:
             def matches(magic):
                 return magic.startswith(bare_text)
 
         comp = [ pre2+m for m in cell_magics if matches(m)]
         if not text.startswith(pre2):
             comp += [ pre+m for m in line_magics if matches(m)]
 
         return comp
 
     def magic_config_matches(self, text:str) -> List[str]:
         """ Match class names and attributes for %config magic """
         texts = text.strip().split()
 
         if len(texts) > 0 and (texts[0] == 'config' or texts[0] == '%config'):
             # get all configuration classes
             classes = sorted(set([ c for c in self.shell.configurables
                                    if c.__class__.class_traits(config=True)
                                    ]), key=lambda x: x.__class__.__name__)
             classnames = [ c.__class__.__name__ for c in classes ]
 
             # return all classnames if config or %config is given
             if len(texts) == 1:
                 return classnames
 
             # match classname
             classname_texts = texts[1].split('.')
             classname = classname_texts[0]
             classname_matches = [ c for c in classnames
                                   if c.startswith(classname) ]
 
             # return matched classes or the matched class with attributes
             if texts[1].find('.') < 0:
                 return classname_matches
             elif len(classname_matches) == 1 and \
                             classname_matches[0] == classname:
                 cls = classes[classnames.index(classname)].__class__
                 help = cls.class_get_help()
                 # strip leading '--' from cl-args:
                 help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
                 return [ attr.split('=')[0]
                          for attr in help.strip().splitlines()
                          if attr.startswith(texts[1]) ]
         return []
 
     def magic_color_matches(self, text:str) -> List[str] :
         """ Match color schemes for %colors magic"""
         texts = text.split()
         if text.endswith(' '):
             # .split() strips off the trailing whitespace. Add '' back
             # so that: '%colors ' -> ['%colors', '']
             texts.append('')
 
         if len(texts) == 2 and (texts[0] == 'colors' or texts[0] == '%colors'):
             prefix = texts[1]
             return [ color for color in InspectColors.keys()
                      if color.startswith(prefix) ]
         return []
 
     def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str):
         """
 
         Return a list of :any:`jedi.api.Completions` object from a ``text`` and
         cursor position.
 
         Parameters
         ----------
         cursor_column : int
             column position of the cursor in ``text``, 0-indexed.
         cursor_line : int
             line position of the cursor in ``text``, 0-indexed
         text : str
             text to complete
 
         Debugging
         ---------
 
         If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
         object containing a string with the Jedi debug information attached.
         """
         namespaces = [self.namespace]
         if self.global_namespace is not None:
             namespaces.append(self.global_namespace)
 
         completion_filter = lambda x:x
         offset = cursor_to_position(text, cursor_line, cursor_column)
         # filter output if we are completing for object members
         if offset:
             pre = text[offset-1]
             if pre == '.':
                 if self.omit__names == 2:
                     completion_filter = lambda c:not c.name.startswith('_')
                 elif self.omit__names == 1:
                     completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__'))
                 elif self.omit__names == 0:
                     completion_filter = lambda x:x
                 else:
                     raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
 
         interpreter = jedi.Interpreter(
             text[:offset], namespaces, column=cursor_column, line=cursor_line + 1)
         try_jedi = True
 
         try:
             # should we check the type of the node is Error ?
             try:
                 # jedi < 0.11
                 from jedi.parser.tree import ErrorLeaf
             except ImportError:
                 # jedi >= 0.11
                 from parso.tree import ErrorLeaf
 
             next_to_last_tree = interpreter._get_module().tree_node.children[-2]
             completing_string = False
             if isinstance(next_to_last_tree, ErrorLeaf):
                 completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"}
             # if we are in a string jedi is likely not the right candidate for
             # now. Skip it.
             try_jedi = not completing_string
         except Exception as e:
             # many of things can go wrong, we are using private API just don't crash.
             if self.debug:
                 print("Error detecting if completing a non-finished string :", e, '|')
 
         if not try_jedi:
             return []
         try:
             return filter(completion_filter, interpreter.completions())
         except Exception as e:
             if self.debug:
                 return [_FakeJediCompletion('Oops Jedi has crashed, please report a bug with the following:\n"""\n%s\ns"""' % (e))]
             else:
                 return []
 
     def python_matches(self, text):
         """Match attributes or global python names"""
         if "." in text:
             try:
                 matches = self.attr_matches(text)
                 if text.endswith('.') and self.omit__names:
                     if self.omit__names == 1:
                         # true if txt is _not_ a __ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'.*\.__.*?__',txt) is None)
                     else:
                         # true if txt is _not_ a _ name, false otherwise:
                         no__name = (lambda txt:
                                     re.match(r'\._.*?',txt[txt.rindex('.'):]) is None)
                     matches = filter(no__name, matches)
             except NameError:
                 # catches <undefined attributes>.<tab>
                 matches = []
         else:
             matches = self.global_matches(text)
         return matches
 
     def _default_arguments_from_docstring(self, doc):
         """Parse the first line of docstring for call signature.
 
         Docstring should be of the form 'min(iterable[, key=func])\n'.
         It can also parse cython docstring of the form
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)'.
         """
         if doc is None:
             return []
 
         #care only the firstline
         line = doc.lstrip().splitlines()[0]
 
         #p = re.compile(r'^[\w|\s.]+\(([^)]*)\).*')
         #'min(iterable[, key=func])\n' -> 'iterable[, key=func]'
         sig = self.docstring_sig_re.search(line)
         if sig is None:
             return []
         # iterable[, key=func]' -> ['iterable[' ,' key=func]']
         sig = sig.groups()[0].split(',')
         ret = []
         for s in sig:
             #re.compile(r'[\s|\[]*(\w+)(?:\s*=\s*.*)')
             ret += self.docstring_kwd_re.findall(s)
         return ret
 
     def _default_arguments(self, obj):
         """Return the list of default arguments of obj if it is callable,
         or empty list otherwise."""
         call_obj = obj
         ret = []
         if inspect.isbuiltin(obj):
             pass
         elif not (inspect.isfunction(obj) or inspect.ismethod(obj)):
             if inspect.isclass(obj):
                 #for cython embedsignature=True the constructor docstring
                 #belongs to the object itself not __init__
                 ret += self._default_arguments_from_docstring(
                             getattr(obj, '__doc__', ''))
                 # for classes, check for __init__,__new__
                 call_obj = (getattr(obj, '__init__', None) or
                        getattr(obj, '__new__', None))
             # for all others, check if they are __call__able
             elif hasattr(obj, '__call__'):
                 call_obj = obj.__call__
         ret += self._default_arguments_from_docstring(
                  getattr(call_obj, '__doc__', ''))
 
         _keeps = (inspect.Parameter.KEYWORD_ONLY,
                   inspect.Parameter.POSITIONAL_OR_KEYWORD)
 
         try:
             sig = inspect.signature(call_obj)
             ret.extend(k for k, v in sig.parameters.items() if
                        v.kind in _keeps)
         except ValueError:
             pass
 
         return list(set(ret))
 
     def python_func_kw_matches(self,text):
         """Match named parameters (kwargs) of the last open function"""
 
         if "." in text: # a parameter cannot be dotted
             return []
         try: regexp = self.__funcParamsRegex
         except AttributeError:
             regexp = self.__funcParamsRegex = re.compile(r'''
                 '.*?(?<!\\)' |    # single quoted strings or
                 ".*?(?<!\\)" |    # double quoted strings or
                 \w+          |    # identifier
                 \S                # other characters
                 ''', re.VERBOSE | re.DOTALL)
         # 1. find the nearest identifier that comes before an unclosed
         # parenthesis before the cursor
         # e.g. for "foo (1+bar(x), pa<cursor>,a=1)", the candidate is "foo"
         tokens = regexp.findall(self.text_until_cursor)
         iterTokens = reversed(tokens); openPar = 0
 
         for token in iterTokens:
             if token == ')':
                 openPar -= 1
             elif token == '(':
                 openPar += 1
                 if openPar > 0:
                     # found the last unclosed parenthesis
                     break
         else:
             return []
         # 2. Concatenate dotted names ("foo.bar" for "foo.bar(x, pa" )
         ids = []
         isId = re.compile(r'\w+$').match
 
         while True:
             try:
                 ids.append(next(iterTokens))
                 if not isId(ids[-1]):
                     ids.pop(); break
                 if not next(iterTokens) == '.':
                     break
             except StopIteration:
                 break
 
         # Find all named arguments already assigned to, as to avoid suggesting
         # them again
         usedNamedArgs = set()
         par_level = -1
         for token, next_token in zip(tokens, tokens[1:]):
             if token == '(':
                 par_level += 1
             elif token == ')':
                 par_level -= 1
 
             if par_level != 0:
                 continue
 
             if next_token != '=':
                 continue
 
             usedNamedArgs.add(token)
 
         # lookup the candidate callable matches either using global_matches
         # or attr_matches for dotted names
         if len(ids) == 1:
             callableMatches = self.global_matches(ids[0])
         else:
             callableMatches = self.attr_matches('.'.join(ids[::-1]))
         argMatches = []
         for callableMatch in callableMatches:
             try:
                 namedArgs = self._default_arguments(eval(callableMatch,
                                                         self.namespace))
             except:
                 continue
 
             # Remove used named arguments from the list, no need to show twice
             for namedArg in set(namedArgs) - usedNamedArgs:
                 if namedArg.startswith(text):
                     argMatches.append(u"%s=" %namedArg)
         return argMatches
 
     def dict_key_matches(self, text):
         "Match string keys in a dictionary, after e.g. 'foo[' "
         def get_keys(obj):
             # Objects can define their own completions by defining an
             # _ipy_key_completions_() method.
             method = get_real_method(obj, '_ipython_key_completions_')
             if method is not None:
                 return method()
 
             # Special case some common in-memory dict-like types
             if isinstance(obj, dict) or\
                _safe_isinstance(obj, 'pandas', 'DataFrame'):
                 try:
                     return list(obj.keys())
                 except Exception:
                     return []
             elif _safe_isinstance(obj, 'numpy', 'ndarray') or\
                  _safe_isinstance(obj, 'numpy', 'void'):
                 return obj.dtype.names or []
             return []
 
         try:
             regexps = self.__dict_key_regexps
         except AttributeError:
             dict_key_re_fmt = r'''(?x)
             (  # match dict-referring expression wrt greedy setting
                 %s
             )
             \[   # open bracket
             \s*  # and optional whitespace
             ([uUbB]?  # string prefix (r not handled)
                 (?:   # unclosed string
                     '(?:[^']|(?<!\\)\\')*
                 |
                     "(?:[^"]|(?<!\\)\\")*
                 )
             )?
             $
             '''
             regexps = self.__dict_key_regexps = {
-                False: re.compile(dict_key_re_fmt % '''
+                False: re.compile(dict_key_re_fmt % r'''
                                   # identifiers separated by .
                                   (?!\d)\w+
                                   (?:\.(?!\d)\w+)*
                                   '''),
                 True: re.compile(dict_key_re_fmt % '''
                                  .+
                                  ''')
             }
 
         match = regexps[self.greedy].search(self.text_until_cursor)
         if match is None:
             return []
 
         expr, prefix = match.groups()
         try:
             obj = eval(expr, self.namespace)
         except Exception:
             try:
                 obj = eval(expr, self.global_namespace)
             except Exception:
                 return []
 
         keys = get_keys(obj)
         if not keys:
             return keys
         closing_quote, token_offset, matches = match_dict_keys(keys, prefix, self.splitter.delims)
         if not matches:
             return matches
         
         # get the cursor position of
         # - the text being completed
         # - the start of the key text
         # - the start of the completion
         text_start = len(self.text_until_cursor) - len(text)
         if prefix:
             key_start = match.start(2)
             completion_start = key_start + token_offset
         else:
             key_start = completion_start = match.end()
         
         # grab the leading prefix, to make sure all completions start with `text`
         if text_start > key_start:
             leading = ''
         else:
             leading = text[text_start:completion_start]
         
         # the index of the `[` character
         bracket_idx = match.end(1)
 
         # append closing quote and bracket as appropriate
         # this is *not* appropriate if the opening quote or bracket is outside
         # the text given to this method
         suf = ''
         continuation = self.line_buffer[len(self.text_until_cursor):]
         if key_start > text_start and closing_quote:
             # quotes were opened inside text, maybe close them
             if continuation.startswith(closing_quote):
                 continuation = continuation[len(closing_quote):]
             else:
                 suf += closing_quote
         if bracket_idx > text_start:
             # brackets were opened inside text, maybe close them
             if not continuation.startswith(']'):
                 suf += ']'
         
         return [leading + k + suf for k in matches]
 
     def unicode_name_matches(self, text):
         u"""Match Latex-like syntax for unicode characters base
         on the name of the character.
         
         This does  ``\\GREEK SMALL LETTER ETA`` -> ``η``
 
         Works only on valid python 3 identifier, or on combining characters that
         will combine to form a valid identifier.
         
         Used on Python 3 only.
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
             s = text[slashpos+1:]
             try :
                 unic = unicodedata.lookup(s)
                 # allow combining chars
                 if ('a'+unic).isidentifier():
                     return '\\'+s,[unic]
             except KeyError:
                 pass
         return u'', []
 
 
     def latex_matches(self, text):
         u"""Match Latex syntax for unicode characters.
         
         This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
 
         Used on Python 3 only.
         """
         slashpos = text.rfind('\\')
         if slashpos > -1:
             s = text[slashpos:]
             if s in latex_symbols:
                 # Try to complete a full latex symbol to unicode
                 # \\alpha -> α
                 return s, [latex_symbols[s]]
             else:
                 # If a user has partially typed a latex symbol, give them
                 # a full list of options \al -> [\aleph, \alpha]
                 matches = [k for k in latex_symbols if k.startswith(s)]
                 return s, matches
         return u'', []
 
     def dispatch_custom_completer(self, text):
         if not self.custom_completers:
             return
 
         line = self.line_buffer
         if not line.strip():
             return None
 
         # Create a little structure to pass all the relevant information about
         # the current completion to any custom completer.
         event = SimpleNamespace()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
         event.command = cmd
         event.text_until_cursor = self.text_until_cursor
 
         # for foo etc, try also to find completer for %foo
         if not cmd.startswith(self.magic_escape):
             try_magic = self.custom_completers.s_matches(
                 self.magic_escape + cmd)
         else:
             try_magic = []
 
         for c in itertools.chain(self.custom_completers.s_matches(cmd),
                  try_magic,
                  self.custom_completers.flat_matches(self.text_until_cursor)):
             try:
                 res = c(event)
                 if res:
                     # first, try case sensitive match
                     withcase = [r for r in res if r.startswith(text)]
                     if withcase:
                         return withcase
                     # if none, then case insensitive ones are ok too
                     text_low = text.lower()
                     return [r for r in res if r.lower().startswith(text_low)]
             except TryNext:
                 pass
             except KeyboardInterrupt:
                 """
                 If custom completer take too long,
                 let keyboard interrupt abort and return nothing.
                 """
                 break
 
         return None
 
     def completions(self, text: str, offset: int)->Iterator[Completion]:
         """
         Returns an iterator over the possible completions
 
         .. warning:: Unstable
             
             This function is unstable, API may change without warning.
             It will also raise unless use in proper context manager.
 
         Parameters
         ----------
         
         text:str
             Full text of the current input, multi line string.
         offset:int
             Integer representing the position of the cursor in ``text``. Offset
             is 0-based indexed.
 
         Yields
         ------
             :any:`Completion` object
 
 
         The cursor on a text can either be seen as being "in between"
         characters or "On" a character depending on the interface visible to
         the user. For consistency the cursor being on "in between" characters X
         and Y is equivalent to the cursor being "on" character Y, that is to say
         the character the cursor is on is considered as being after the cursor.
 
         Combining characters may span more that one position in the
         text.
 
 
         .. note::
 
             If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--``
             fake Completion token to distinguish completion returned by Jedi
             and usual IPython completion.
 
         .. note::
             
             Completions are not completely deduplicated yet. If identical
             completions are coming from different sources this function does not
             ensure that each completion object will only be present once.
         """
         warnings.warn("_complete is a provisional API (as of IPython 6.0). "
                       "It may change without warnings. "
                       "Use in corresponding context manager.",
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
         seen = set()
         try:
             for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
                 if c and (c in seen):
                     continue
                 yield c
                 seen.add(c)
         except KeyboardInterrupt:
             """if completions take too long and users send keyboard interrupt,
             do not crash and return ASAP. """
             pass
 
     def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]:
         """
         Core completion module.Same signature as :any:`completions`, with the
         extra `timeout` parameter (in seconds).
 
 
         Computing jedi's completion ``.type`` can be quite expensive (it is a
         lazy property) and can require some warm-up, more warm up than just
         computing the ``name`` of a completion. The warm-up can be :
 
             - Long warm-up the first time a module is encountered after
             install/update: actually build parse/inference tree.
 
             - first time the module is encountered in a session: load tree from
             disk.
 
         We don't want to block completions for tens of seconds so we give the
         completer a "budget" of ``_timeout`` seconds per invocation to compute
         completions types, the completions that have not yet been computed will
         be marked as "unknown" an will have a chance to be computed next round
         are things get cached.
 
         Keep in mind that Jedi is not the only thing treating the completion so
         keep the timeout short-ish as if we take more than 0.3 second we still
         have lots of processing to do.
 
         """
         deadline = time.monotonic() + _timeout
 
 
         before = full_text[:offset]
         cursor_line, cursor_column = position_to_cursor(full_text, offset)
 
         matched_text, matches, matches_origin, jedi_matches = self._complete(
             full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column)
 
         iter_jm = iter(jedi_matches)
         if _timeout:
             for jm in iter_jm:
                 try:
                     type_ = jm.type
                 except Exception:
                     if self.debug:
                         print("Error in Jedi getting type of ", jm)
                     type_ = None
                 delta = len(jm.name_with_symbols) - len(jm.complete)
                 if type_ == 'function':
                     signature = _make_signature(jm)
                 else:
                     signature = ''
                 yield Completion(start=offset - delta,
                                  end=offset,
                                  text=jm.name_with_symbols,
                                  type=type_,
                                  signature=signature,
                                  _origin='jedi')
 
                 if time.monotonic() > deadline:
                     break
 
         for jm in iter_jm:
             delta = len(jm.name_with_symbols) - len(jm.complete)
             yield Completion(start=offset - delta,
                              end=offset,
                              text=jm.name_with_symbols,
                              type='<unknown>',  # don't compute type for speed
                              _origin='jedi',
                              signature='')
 
 
         start_offset = before.rfind(matched_text)
 
         # TODO:
         # Suppress this, right now just for debug.
         if jedi_matches and matches and self.debug:
             yield Completion(start=start_offset, end=offset, text='--jedi/ipython--',
                              _origin='debug', type='none', signature='')
 
         # I'm unsure if this is always true, so let's assert and see if it
         # crash
         assert before.endswith(matched_text)
         for m, t in zip(matches, matches_origin):
             yield Completion(start=start_offset, end=offset, text=m, _origin=t, signature='', type='<unknown>')
 
 
     def complete(self, text=None, line_buffer=None, cursor_pos=None):
         """Find completions for the given text and line context.
 
         Note that both the text and the line_buffer are optional, but at least
         one of them must be given.
 
         Parameters
         ----------
           text : string, optional
             Text to perform the completion on.  If not given, the line buffer
             is split using the instance's CompletionSplitter object.
 
           line_buffer : string, optional
             If not given, the completer attempts to obtain the current line
             buffer via readline.  This keyword allows clients which are
             requesting for text completions in non-readline contexts to inform
             the completer of the entire text.
 
           cursor_pos : int, optional
             Index of the cursor in the full line buffer.  Should be provided by
             remote frontends where kernel has no access to frontend state.
 
         Returns
         -------
         text : str
           Text that was actually used in the completion.
 
         matches : list
           A list of completion matches.
 
 
         .. note::
 
             This API is likely to be deprecated and replaced by
             :any:`IPCompleter.completions` in the future.
 
 
         """
         warnings.warn('`Completer.complete` is pending deprecation since '
                 'IPython 6.0 and will be replaced by `Completer.completions`.',
                       PendingDeprecationWarning)
         # potential todo, FOLD the 3rd throw away argument of _complete
         # into the first 2 one.
         return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2]
 
     def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
                   full_text=None) -> Tuple[str, List[str], List[str], Iterable[_FakeJediCompletion]]:
         """
 
         Like complete but can also returns raw jedi completions as well as the
         origin of the completion text. This could (and should) be made much
         cleaner but that will be simpler once we drop the old (and stateful)
         :any:`complete` API.
 
 
         With current provisional API, cursor_pos act both (depending on the
         caller) as the offset in the ``text`` or ``line_buffer``, or as the
         ``column`` when passing multiline strings this could/should be renamed
         but would add extra noise.
         """
 
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
             cursor_pos = len(line_buffer) if text is None else len(text)
 
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
         # if text is either None or an empty string, rely on the line buffer
         if (not line_buffer) and full_text:
             line_buffer = full_text.split('\n')[cursor_line]
         if not text:
             text = self.splitter.split_line(line_buffer, cursor_pos)
 
         if self.backslash_combining_completions:
             # allow deactivation of these on windows.
             base_text = text if not line_buffer else line_buffer[:cursor_pos]
             latex_text, latex_matches = self.latex_matches(base_text)
             if latex_matches:
                 return latex_text, latex_matches, ['latex_matches']*len(latex_matches), ()
             name_text = ''
             name_matches = []
             for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches):
                 name_text, name_matches = meth(base_text)
                 if name_text:
                     return name_text, name_matches[:MATCHES_LIMIT], \
                            [meth.__qualname__]*min(len(name_matches), MATCHES_LIMIT), ()
         
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
             line_buffer = text
 
         self.line_buffer = line_buffer
         self.text_until_cursor = self.line_buffer[:cursor_pos]
 
         # Do magic arg matches
         for matcher in self.magic_arg_matchers:
             matches = list(matcher(line_buffer))[:MATCHES_LIMIT]
             if matches:
                 origins = [matcher.__qualname__] * len(matches)
                 return text, matches, origins, ()
 
         # Start with a clean slate of completions
         matches = []
         custom_res = self.dispatch_custom_completer(text)
         # FIXME: we should extend our api to return a dict with completions for
         # different types of objects.  The rlcomplete() method could then
         # simply collapse the dict into a list for readline, but we'd have
         # richer completion semantics in other environments.
         completions = ()
         if self.use_jedi:
             if not full_text:
                 full_text = line_buffer
             completions = self._jedi_matches(
                 cursor_pos, cursor_line, full_text)
         if custom_res is not None:
             # did custom completers produce something?
             matches = [(m, 'custom') for m in custom_res]
         else:
             # Extend the list of completions with the results of each
             # matcher, so we return results to the user from all
             # namespaces.
             if self.merge_completions:
                 matches = []
                 for matcher in self.matchers:
                     try:
                         matches.extend([(m, matcher.__qualname__)
                                         for m in matcher(text)])
                     except:
                         # Show the ugly traceback if the matcher causes an
                         # exception, but do NOT crash the kernel!
                         sys.excepthook(*sys.exc_info())
             else:
                 for matcher in self.matchers:
                     matches = [(m, matcher.__qualname__)
                                for m in matcher(text)]
                     if matches:
                         break
         seen = set()
         filtered_matches = set()
         for m in matches:
             t, c = m
             if t not in seen:
                 filtered_matches.add(m)
                 seen.add(t)
 
         _filtered_matches = sorted(
             set(filtered_matches), key=lambda x: completions_sorting_key(x[0]))\
             [:MATCHES_LIMIT]
 
         _matches = [m[0] for m in _filtered_matches]
         origins = [m[1] for m in _filtered_matches]
 
         self.matches = _matches
 
         return text, _matches, origins, completions

IPython/core/debugger.py

@@ -1,645 +1,645 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
 changes. Licensing should therefore be under the standard Python terms.  For
 details on the PSF (Python Software Foundation) standard license, see:
 
 https://docs.python.org/2/license.html
 """
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 
 import bdb
 import functools
 import inspect
 import linecache
 import sys
 import warnings
 import re
 
 from IPython import get_ipython
 from IPython.utils import PyColorize
 from IPython.utils import coloransi, py3compat
 from IPython.core.excolors import exception_colors
 from IPython.testing.skipdoctest import skip_doctest
 
 
 prompt = 'ipdb> '
 
 #We have to check this directly from sys.argv, config struct not yet available
 from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 
 def make_arrow(pad):
     """generate the leading arrow in front of traceback or debugger"""
     if pad >= 2:
         return '-'*(pad-2) + '> '
     elif pad == 1:
         return '>'
     return ''
 
 
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
                   DeprecationWarning, stacklevel=2)
     if et==bdb.BdbQuit:
         print('Exiting Debugger.')
     elif excepthook is not None:
         excepthook(et, ev, tb)
     else:
         # Backwards compatibility. Raise deprecation warning?
         BdbQuit_excepthook.excepthook_ori(et,ev,tb)
 
 
 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
     warnings.warn(
         "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
         DeprecationWarning, stacklevel=2)
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """
     DEPRECATED
 
     Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
     @skip_doctest
     def __init__(self, colors=None):
         """
         DEPRECATED
 
         Create a local debugger instance.
 
         Parameters
         ----------
 
         colors : str, optional
             The name of the color scheme to use, it must be one of IPython's
             valid color schemes.  If not given, the function will default to
             the current IPython scheme when running inside IPython, and to
             'NoColor' otherwise.
 
         Examples
         --------
         ::
 
             from IPython.core.debugger import Tracer; debug_here = Tracer()
 
         Later in your code::
 
             debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
         warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
                       "`IPython.core.debugger.Pdb.set_trace()`",
                       DeprecationWarning, stacklevel=2)
 
         ip = get_ipython()
         if ip is None:
             # Outside of ipython, we set our own exception hook manually
             sys.excepthook = functools.partial(BdbQuit_excepthook,
                                                excepthook=sys.excepthook)
             def_colors = 'NoColor'
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             try:
                 from reprlib import aRepr  # Py 3
             except ImportError:
                 from repr import aRepr  # Py 2
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
-RGX_EXTRA_INDENT = re.compile('(?<=\n)\s+')
+RGX_EXTRA_INDENT = re.compile(r'(?<=\n)\s+')
 
 
 def strip_indentation(multiline_string):
     return RGX_EXTRA_INDENT.sub('', multiline_string)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = strip_indentation(old_fn.__doc__) + additional_text
     return wrapper
 
 
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
     This function never raises an IOError exception: if the file can't be
     read, it simply returns an empty list."""
 
     try:
         outfile = open(fname)
     except IOError:
         return []
     else:
         out = outfile.readlines()
         outfile.close()
         return out
 
 
 class Pdb(OldPdb):
     """Modified Pdb class, does not load readline.
 
     for a standalone version that uses prompt_toolkit, see
     `IPython.terminal.debugger.TerminalPdb` and
     `IPython.terminal.debugger.set_trace()`
     """
 
     def __init__(self, color_scheme=None, completekey=None,
                  stdin=None, stdout=None, context=5):
 
         # Parent constructor:
         try:
             self.context = int(context)
             if self.context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
 
         OldPdb.__init__(self, completekey, stdin, stdout)
 
         # IPython changes...
         self.shell = get_ipython()
 
         if self.shell is None:
             save_main = sys.modules['__main__']
             # No IPython instance running, we must create one
             from IPython.terminal.interactiveshell import \
                 TerminalInteractiveShell
             self.shell = TerminalInteractiveShell.instance()
             # needed by any code which calls __import__("__main__") after
             # the debugger was entered. See also #9941.
             sys.modules['__main__'] = save_main 
 
         if color_scheme is not None:
             warnings.warn(
                 "The `color_scheme` argument is deprecated since version 5.1",
                 DeprecationWarning, stacklevel=2)
         else:
             color_scheme = self.shell.colors
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.prompt = C.NoColor
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.prompt = C.Green
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.prompt = C.Blue
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         cst['Neutral'].colors.prompt = C.Blue
         cst['Neutral'].colors.breakpoint_enabled = C.LightRed
         cst['Neutral'].colors.breakpoint_disabled = C.Red
 
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser(style=color_scheme)
         self.set_colors(color_scheme)
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
         self.parser.style = scheme
 
     def interaction(self, frame, traceback):
         try:
             OldPdb.interaction(self, frame, traceback)
         except KeyboardInterrupt:
             sys.stdout.write('\n' + self.shell.get_exception_only())
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions=self.old_all_completions
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def print_stack_trace(self, context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
         try:
             for frame_lineno in self.stack:
                 self.print_stack_entry(frame_lineno, context=context)
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
                           context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
         print(self.format_stack_entry(frame_lineno, '', context))
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 print("Context must be a positive integer")
         except (TypeError, ValueError):
                 print("Context must be a positive integer")
         try:
             import reprlib  # Py 3
         except ImportError:
             import repr as reprlib  # Py 2
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         if '__return__' in frame.f_locals:
             rv = frame.f_locals['__return__']
             #return_value += '->'
             return_value += reprlib.repr(rv) + '\n'
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ''
         if func != '?':
             if '__args__' in frame.f_locals:
                 args = reprlib.repr(frame.f_locals['__args__'])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append('  ')
         ret.append(u'%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
         lines = linecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i,line in enumerate(lines):
             show_arrow = (start + 1 + i == lineno)
             linetpl = (frame is self.curframe or show_arrow) \
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
                                           start + 1 + i, line,
                                           arrow = show_arrow) )
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
         bp_mark = ""
         bp_mark_color = ""
 
         new_line, err = self.parser.format2(line, 'str')
         if not err:
             line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             num = '%s%s' % (make_arrow(pad), str(lineno))
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
 
         return tpl_line % (bp_mark_color + bp_mark, num, line)
 
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = linecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
                 else:
                     line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src))
 
         except KeyboardInterrupt:
             pass
 
     def do_list(self, arg):
         """Print lines of code from the current stack frame
         """
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg))
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def getsourcelines(self, obj):
         lines, lineno = inspect.findsource(obj)
         if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
             # must be a module frame: do not try to cut a block out of it
             return lines, 1
         elif inspect.ismodule(obj):
             return lines, 1
         return inspect.getblock(lines[lineno:]), lineno+1
 
     def do_longlist(self, arg):
         """Print lines of code from the current stack frame.
 
         Shows more lines than 'list' does.
         """
         self.lastcmd = 'longlist'
         try:
             lines, lineno = self.getsourcelines(self.curframe)
         except OSError as err:
             self.error(err)
             return
         last = lineno + len(lines)
         self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
     do_ll = do_longlist
 
     def do_debug(self, arg):
         """debug code
         Enter a recursive debugger that steps through the code
         argument (which is an arbitrary expression or statement to be
         executed in the current environment).
         """
         sys.settrace(None)
         globals = self.curframe.f_globals
         locals = self.curframe_locals
         p = self.__class__(completekey=self.completekey,
                            stdin=self.stdin, stdout=self.stdout)
         p.use_rawinput = self.use_rawinput
         p.prompt = "(%s) " % self.prompt.strip()
         self.message("ENTERING RECURSIVE DEBUGGER")
         sys.call_tracing(p.run, (arg, globals, locals))
         self.message("LEAVING RECURSIVE DEBUGGER")
         sys.settrace(self.trace_dispatch)
         self.lastcmd = p.lastcmd
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
 
     def do_where(self, arg):
         """w(here)
         Print a stack trace, with the most recent frame at the bottom.
         An arrow indicates the "current frame", which determines the
         context of most commands. 'bt' is an alias for this command.
 
         Take a number as argument as an (optional) number of context line to
         print"""
         if arg:
             context = int(arg)
             self.print_stack_trace(context)
         else:
             self.print_stack_trace()
 
     do_w = do_where
 
 
 def set_trace(frame=None):
     """
     Start debugging from `frame`.
 
     If frame is not specified, debugging starts from caller's frame.
     """
     Pdb().set_trace(frame or sys._getframe().f_back)

IPython/core/inputsplitter.py

@@ -1,766 +1,766 b''
 """Input handling and transformation machinery.
 
 The first class in this module, :class:`InputSplitter`, is designed to tell when
 input from a line-oriented frontend is complete and should be executed, and when
 the user should be prompted for another line of code instead. The name 'input
 splitter' is largely for historical reasons.
 
 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
 with full support for the extended IPython syntax (magics, system calls, etc).
 The code to actually do these transformations is in :mod:`IPython.core.inputtransformer`.
 :class:`IPythonInputSplitter` feeds the raw code to the transformers in order
 and stores the results.
 
 For more details, see the class docstrings below.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 import ast
 import codeop
 import io
 import re
 import sys
 import tokenize
 import warnings
 
 from IPython.utils.py3compat import cast_unicode
 from IPython.core.inputtransformer import (leading_indent,
                                            classic_prompt,
                                            ipy_prompt,
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
                                            escaped_commands,
                                            assign_from_magic,
                                            assign_from_system,
                                            assemble_python_lines,
                                            )
 
 # These are available in this module for backwards compatibility.
 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
                                         ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
                                         ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 # FIXME: These are general-purpose utilities that later can be moved to the
 # general ward.  Kept here for now because we're being very strict about test
 # coverage with this code, and this lets us ensure that we keep 100% coverage
 # while developing.
 
 # compiled regexps for autoindent management
 dedent_re = re.compile('|'.join([
     r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
     r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
     r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
     r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
     r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
     r'^\s+break\s*$', # break (optionally followed by trailing spaces)
     r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
 ]))
 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
 
 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
 # before pure comments
-comment_line_re = re.compile('^\s*\#')
+comment_line_re = re.compile(r'^\s*\#')
 
 
 def num_ini_spaces(s):
     """Return the number of initial spaces in a string.
 
     Note that tabs are counted as a single space.  For now, we do *not* support
     mixing of tabs and spaces in the user's input.
 
     Parameters
     ----------
     s : string
 
     Returns
     -------
     n : int
     """
 
     ini_spaces = ini_spaces_re.match(s)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 # Fake token types for partial_tokenize:
 INCOMPLETE_STRING = tokenize.N_TOKENS
 IN_MULTILINE_STATEMENT = tokenize.N_TOKENS + 1
 
 # The 2 classes below have the same API as TokenInfo, but don't try to look up
 # a token type name that they won't find.
 class IncompleteString:
     type = exact_type = INCOMPLETE_STRING
     def __init__(self, s, start, end, line):
         self.s = s
         self.start = start
         self.end = end
         self.line = line
 
 class InMultilineStatement:
     type = exact_type = IN_MULTILINE_STATEMENT
     def __init__(self, pos, line):
         self.s = ''
         self.start = self.end = pos
         self.line = line
 
 def partial_tokens(s):
     """Iterate over tokens from a possibly-incomplete string of code.
 
     This adds two special token types: INCOMPLETE_STRING and
     IN_MULTILINE_STATEMENT. These can only occur as the last token yielded, and
     represent the two main ways for code to be incomplete.
     """
     readline = io.StringIO(s).readline
     token = tokenize.TokenInfo(tokenize.NEWLINE, '', (1, 0), (1, 0), '')
     try:
         for token in tokenize.generate_tokens(readline):
             yield token
     except tokenize.TokenError as e:
         # catch EOF error
         lines = s.splitlines(keepends=True)
         end = len(lines), len(lines[-1])
         if 'multi-line string' in e.args[0]:
             l, c = start = token.end
             s = lines[l-1][c:] + ''.join(lines[l:])
             yield IncompleteString(s, start, end, lines[-1])
         elif 'multi-line statement' in e.args[0]:
             yield InMultilineStatement(end, lines[-1])
         else:
             raise
 
 def find_next_indent(code):
     """Find the number of spaces for the next line of indentation"""
     tokens = list(partial_tokens(code))
     if tokens[-1].type == tokenize.ENDMARKER:
         tokens.pop()
     if not tokens:
         return 0
     while (tokens[-1].type in {tokenize.DEDENT, tokenize.NEWLINE, tokenize.COMMENT}):
         tokens.pop()
 
     if tokens[-1].type == INCOMPLETE_STRING:
         # Inside a multiline string
         return 0
 
     # Find the indents used before
     prev_indents = [0]
     def _add_indent(n):
         if n != prev_indents[-1]:
             prev_indents.append(n)
 
     tokiter = iter(tokens)
     for tok in tokiter:
         if tok.type in {tokenize.INDENT, tokenize.DEDENT}:
             _add_indent(tok.end[1])
         elif (tok.type == tokenize.NL):
             try:
                 _add_indent(next(tokiter).start[1])
             except StopIteration:
                 break
 
     last_indent = prev_indents.pop()
 
     # If we've just opened a multiline statement (e.g. 'a = ['), indent more
     if tokens[-1].type == IN_MULTILINE_STATEMENT:
         if tokens[-2].exact_type in {tokenize.LPAR, tokenize.LSQB, tokenize.LBRACE}:
             return last_indent + 4
         return last_indent
 
     if tokens[-1].exact_type == tokenize.COLON:
         # Line ends with colon - indent
         return last_indent + 4
 
     if last_indent:
         # Examine the last line for dedent cues - statements like return or
         # raise which normally end a block of code.
         last_line_starts = 0
         for i, tok in enumerate(tokens):
             if tok.type == tokenize.NEWLINE:
                 last_line_starts = i + 1
 
         last_line_tokens = tokens[last_line_starts:]
         names = [t.string for t in last_line_tokens if t.type == tokenize.NAME]
         if names and names[0] in {'raise', 'return', 'pass', 'break', 'continue'}:
             # Find the most recent indentation less than the current level
             for indent in reversed(prev_indents):
                 if indent < last_indent:
                     return indent
 
     return last_indent
 
 
 def last_blank(src):
     """Determine if the input source ends in a blank.
 
     A blank is either a newline or a line consisting of whitespace.
 
     Parameters
     ----------
     src : string
       A single or multiline string.
     """
     if not src: return False
     ll  = src.splitlines()[-1]
     return (ll == '') or ll.isspace()
 
 
 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
 
 def last_two_blanks(src):
     """Determine if the input source ends in two blanks.
 
     A blank is either a newline or a line consisting of whitespace.
 
     Parameters
     ----------
     src : string
       A single or multiline string.
     """
     if not src: return False
     # The logic here is tricky: I couldn't get a regexp to work and pass all
     # the tests, so I took a different approach: split the source by lines,
     # grab the last two and prepend '###\n' as a stand-in for whatever was in
     # the body before the last two lines.  Then, with that structure, it's
     # possible to analyze with two regexps.  Not the most elegant solution, but
     # it works.  If anyone tries to change this logic, make sure to validate
     # the whole test suite first!
     new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
     return (bool(last_two_blanks_re.match(new_src)) or
             bool(last_two_blanks_re2.match(new_src)) )
 
 
 def remove_comments(src):
     """Remove all comments from input source.
 
     Note: comments are NOT recognized inside of strings!
 
     Parameters
     ----------
     src : string
       A single or multiline input string.
 
     Returns
     -------
     String with all Python comments removed.
     """
 
     return re.sub('#.*', '', src)
 
 
 def get_input_encoding():
     """Return the default standard input encoding.
 
     If sys.stdin has no encoding, 'ascii' is returned."""
     # There are strange environments for which sys.stdin.encoding is None. We
     # ensure that a valid encoding is returned.
     encoding = getattr(sys.stdin, 'encoding', None)
     if encoding is None:
         encoding = 'ascii'
     return encoding
 
 #-----------------------------------------------------------------------------
 # Classes and functions for normal Python syntax handling
 #-----------------------------------------------------------------------------
 
 class InputSplitter(object):
     r"""An object that can accumulate lines of Python source before execution.
 
     This object is designed to be fed python source line-by-line, using
     :meth:`push`. It will return on each push whether the currently pushed
     code could be executed already. In addition, it provides a method called
     :meth:`push_accepts_more` that can be used to query whether more input
     can be pushed into a single interactive block.
 
     This is a simple example of how an interactive terminal-based client can use
     this tool::
 
         isp = InputSplitter()
         while isp.push_accepts_more():
             indent = ' '*isp.indent_spaces
             prompt = '>>> ' + indent
             line = indent + raw_input(prompt)
             isp.push(line)
         print 'Input source was:\n', isp.source_reset(),
     """
     # A cache for storing the current indentation
     # The first value stores the most recently processed source input
     # The second value is the number of spaces for the current indentation 
     # If self.source matches the first value, the second value is a valid
     # current indentation. Otherwise, the cache is invalid and the indentation
     # must be recalculated.
     _indent_spaces_cache = None, None
     # String, indicating the default input encoding.  It is computed by default
     # at initialization time via get_input_encoding(), but it can be reset by a
     # client with specific knowledge of the encoding.
     encoding = ''
     # String where the current full source input is stored, properly encoded.
     # Reading this attribute is the normal way of querying the currently pushed
     # source code, that has been properly encoded.
     source = ''
     # Code object corresponding to the current source.  It is automatically
     # synced to the source, so it can be queried at any time to obtain the code
     # object; it will be None if the source doesn't compile to valid Python.
     code = None
 
     # Private attributes
 
     # List with lines of input accumulated so far
     _buffer = None
     # Command compiler
     _compile = None
     # Boolean indicating whether the current block is complete
     _is_complete = None
     # Boolean indicating whether the current block has an unrecoverable syntax error
     _is_invalid = False
 
     def __init__(self):
         """Create a new InputSplitter instance.
         """
         self._buffer = []
         self._compile = codeop.CommandCompiler()
         self.encoding = get_input_encoding()
 
     def reset(self):
         """Reset the input buffer and associated state."""
         self._buffer[:] = []
         self.source = ''
         self.code = None
         self._is_complete = False
         self._is_invalid = False
 
     def source_reset(self):
         """Return the input source and perform a full reset.
         """
         out = self.source
         self.reset()
         return out
 
     def check_complete(self, source):
         """Return whether a block of code is ready to execute, or should be continued
         
         This is a non-stateful API, and will reset the state of this InputSplitter.
         
         Parameters
         ----------
         source : string
           Python input code, which can be multiline.
         
         Returns
         -------
         status : str
           One of 'complete', 'incomplete', or 'invalid' if source is not a
           prefix of valid code.
         indent_spaces : int or None
           The number of spaces by which to indent the next line of code. If
           status is not 'incomplete', this is None.
         """
         self.reset()
         try:
             self.push(source)
         except SyntaxError:
             # Transformers in IPythonInputSplitter can raise SyntaxError,
             # which push() will not catch.
             return 'invalid', None
         else:
             if self._is_invalid:
                 return 'invalid', None
             elif self.push_accepts_more():
                 return 'incomplete', self.get_indent_spaces()
             else:
                 return 'complete', None
         finally:
             self.reset()
 
     def push(self, lines):
         """Push one or more lines of input.
 
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not.
 
         Any exceptions generated in compilation are swallowed, but if an
         exception was produced, the method returns True.
 
         Parameters
         ----------
         lines : string
           One or more lines of Python input.
 
         Returns
         -------
         is_complete : boolean
           True if the current input source (the result of the current input
           plus prior inputs) forms a complete Python execution block.  Note that
           this value is also stored as a private attribute (``_is_complete``), so it
           can be queried at any time.
         """
         self._store(lines)
         source = self.source
 
         # Before calling _compile(), reset the code object to None so that if an
         # exception is raised in compilation, we don't mislead by having
         # inconsistent code/source attributes.
         self.code, self._is_complete = None, None
         self._is_invalid = False
 
         # Honor termination lines properly
         if source.endswith('\\\n'):
             return False
 
         try:
             with warnings.catch_warnings():
                 warnings.simplefilter('error', SyntaxWarning)
                 self.code = self._compile(source, symbol="exec")
         # Invalid syntax can produce any of a number of different errors from
         # inside the compiler, so we have to catch them all.  Syntax errors
         # immediately produce a 'ready' block, so the invalid Python can be
         # sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         except (SyntaxError, OverflowError, ValueError, TypeError,
                 MemoryError, SyntaxWarning):
             self._is_complete = True
             self._is_invalid = True
         else:
             # Compilation didn't produce any exceptions (though it may not have
             # given a complete code object)
             self._is_complete = self.code is not None
 
         return self._is_complete
 
     def push_accepts_more(self):
         """Return whether a block of interactive input can accept more input.
 
         This method is meant to be used by line-oriented frontends, who need to
         guess whether a block is complete or not based solely on prior and
         current input lines.  The InputSplitter considers it has a complete
         interactive block and will not accept more input when either:
         
         * A SyntaxError is raised
 
         * The code is complete and consists of a single line or a single
           non-compound statement
 
         * The code is complete and has a blank line at the end
 
         If the current input produces a syntax error, this method immediately
         returns False but does *not* raise the syntax error exception, as
         typically clients will want to send invalid syntax to an execution
         backend which might convert the invalid syntax into valid Python via
         one of the dynamic IPython mechanisms.
         """
 
         # With incomplete input, unconditionally accept more
         # A syntax error also sets _is_complete to True - see push()
         if not self._is_complete:
             #print("Not complete")  # debug
             return True
         
         # The user can make any (complete) input execute by leaving a blank line
         last_line = self.source.splitlines()[-1]
         if (not last_line) or last_line.isspace():
             #print("Blank line")  # debug
             return False
         
         # If there's just a single line or AST node, and we're flush left, as is
         # the case after a simple statement such as 'a=1', we want to execute it
         # straight away.
         if self.get_indent_spaces() == 0:
             if len(self.source.splitlines()) <= 1:
                 return False
             
             try:
                 code_ast = ast.parse(u''.join(self._buffer))
             except Exception:
                 #print("Can't parse AST")  # debug
                 return False
             else:
                 if len(code_ast.body) == 1 and \
                                     not hasattr(code_ast.body[0], 'body'):
                     #print("Simple statement")  # debug
                     return False
 
         # General fallback - accept more code
         return True
 
     def get_indent_spaces(self):
         sourcefor, n = self._indent_spaces_cache
         if sourcefor == self.source:
             return n
 
         # self.source always has a trailing newline
         n = find_next_indent(self.source[:-1])
         self._indent_spaces_cache = (self.source, n)
         return n
 
     # Backwards compatibility. I think all code that used .indent_spaces was
     # inside IPython, but we can leave this here until IPython 7 in case any
     # other modules are using it. -TK, November 2017
     indent_spaces = property(get_indent_spaces)
 
     def _store(self, lines, buffer=None, store='source'):
         """Store one or more lines of input.
 
         If input lines are not newline-terminated, a newline is automatically
         appended."""
 
         if buffer is None:
             buffer = self._buffer
 
         if lines.endswith('\n'):
             buffer.append(lines)
         else:
             buffer.append(lines+'\n')
         setattr(self, store, self._set_source(buffer))
 
     def _set_source(self, buffer):
         return u''.join(buffer)
 
 
 class IPythonInputSplitter(InputSplitter):
     """An input splitter that recognizes all of IPython's special syntax."""
 
     # String with raw, untransformed input.
     source_raw = ''
     
     # Flag to track when a transformer has stored input that it hasn't given
     # back yet.
     transformer_accumulating = False
     
     # Flag to track when assemble_python_lines has stored input that it hasn't
     # given back yet.
     within_python_line = False
 
     # Private attributes
 
     # List with lines of raw input accumulated so far.
     _buffer_raw = None
 
     def __init__(self, line_input_checker=True, physical_line_transforms=None,
                     logical_line_transforms=None, python_line_transforms=None):
         super(IPythonInputSplitter, self).__init__()
         self._buffer_raw = []
         self._validate = True
         
         if physical_line_transforms is not None:
             self.physical_line_transforms = physical_line_transforms
         else:
             self.physical_line_transforms = [
                                              leading_indent(),
                                              classic_prompt(),
                                              ipy_prompt(),
                                              cellmagic(end_on_blank_line=line_input_checker),
                                             ]
         
         self.assemble_logical_lines = assemble_logical_lines()
         if logical_line_transforms is not None:
             self.logical_line_transforms = logical_line_transforms
         else:
             self.logical_line_transforms = [
                                             help_end(),
                                             escaped_commands(),
                                             assign_from_magic(),
                                             assign_from_system(),
                                            ]
         
         self.assemble_python_lines = assemble_python_lines()
         if python_line_transforms is not None:
             self.python_line_transforms = python_line_transforms
         else:
             # We don't use any of these at present
             self.python_line_transforms = []
     
     @property
     def transforms(self):
         "Quick access to all transformers."
         return self.physical_line_transforms + \
             [self.assemble_logical_lines] + self.logical_line_transforms + \
             [self.assemble_python_lines]  + self.python_line_transforms
     
     @property
     def transforms_in_use(self):
         """Transformers, excluding logical line transformers if we're in a
         Python line."""
         t = self.physical_line_transforms[:]
         if not self.within_python_line:
             t += [self.assemble_logical_lines] + self.logical_line_transforms
         return t + [self.assemble_python_lines] + self.python_line_transforms
 
     def reset(self):
         """Reset the input buffer and associated state."""
         super(IPythonInputSplitter, self).reset()
         self._buffer_raw[:] = []
         self.source_raw = ''
         self.transformer_accumulating = False
         self.within_python_line = False
 
         for t in self.transforms:
             try:
                 t.reset()
             except SyntaxError:
                 # Nothing that calls reset() expects to handle transformer
                 # errors
                 pass
     
     def flush_transformers(self):
         def _flush(transform, outs):
             """yield transformed lines
             
             always strings, never None
             
             transform: the current transform
             outs: an iterable of previously transformed inputs.
                  Each may be multiline, which will be passed
                  one line at a time to transform.
             """
             for out in outs:
                 for line in out.splitlines():
                     # push one line at a time
                     tmp = transform.push(line)
                     if tmp is not None:
                         yield tmp
             
             # reset the transform
             tmp = transform.reset()
             if tmp is not None:
                 yield tmp
         
         out = []
         for t in self.transforms_in_use:
             out = _flush(t, out)
         
         out = list(out)
         if out:
             self._store('\n'.join(out))
 
     def raw_reset(self):
         """Return raw input only and perform a full reset.
         """
         out = self.source_raw
         self.reset()
         return out
     
     def source_reset(self):
         try:
             self.flush_transformers()
             return self.source
         finally:
             self.reset()
 
     def push_accepts_more(self):
         if self.transformer_accumulating:
             return True
         else:
             return super(IPythonInputSplitter, self).push_accepts_more()
 
     def transform_cell(self, cell):
         """Process and translate a cell of input.
         """
         self.reset()
         try:
             self.push(cell)
             self.flush_transformers()
             return self.source
         finally:
             self.reset()
 
     def push(self, lines):
         """Push one or more lines of IPython input.
 
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not, after processing
         all input lines for special IPython syntax.
 
         Any exceptions generated in compilation are swallowed, but if an
         exception was produced, the method returns True.
 
         Parameters
         ----------
         lines : string
           One or more lines of Python input.
 
         Returns
         -------
         is_complete : boolean
           True if the current input source (the result of the current input
           plus prior inputs) forms a complete Python execution block.  Note that
           this value is also stored as a private attribute (_is_complete), so it
           can be queried at any time.
         """
 
         # We must ensure all input is pure unicode
         lines = cast_unicode(lines, self.encoding)
         # ''.splitlines() --> [], but we need to push the empty line to transformers
         lines_list = lines.splitlines()
         if not lines_list:
             lines_list = ['']
 
         # Store raw source before applying any transformations to it.  Note
         # that this must be done *after* the reset() call that would otherwise
         # flush the buffer.
         self._store(lines, self._buffer_raw, 'source_raw')
 
         transformed_lines_list = []
         for line in lines_list:
             transformed = self._transform_line(line)
             if transformed is not None:
                 transformed_lines_list.append(transformed)
 
         if transformed_lines_list:
             transformed_lines = '\n'.join(transformed_lines_list)
             return super(IPythonInputSplitter, self).push(transformed_lines)
         else:
             # Got nothing back from transformers - they must be waiting for
             # more input.
             return False
 
     def _transform_line(self, line):
         """Push a line of input code through the various transformers.
         
         Returns any output from the transformers, or None if a transformer
         is accumulating lines.
         
         Sets self.transformer_accumulating as a side effect.
         """
         def _accumulating(dbg):
             #print(dbg)
             self.transformer_accumulating = True
             return None
 
         for transformer in self.physical_line_transforms:
             line = transformer.push(line)
             if line is None:
                 return _accumulating(transformer)
 
         if not self.within_python_line:
             line = self.assemble_logical_lines.push(line)
             if line is None:
                 return _accumulating('acc logical line')
 
             for transformer in self.logical_line_transforms:
                 line = transformer.push(line)
                 if line is None:
                     return _accumulating(transformer)
 
         line = self.assemble_python_lines.push(line)
         if line is None:
             self.within_python_line = True
             return _accumulating('acc python line')
         else:
             self.within_python_line = False
 
         for transformer in self.python_line_transforms:
             line = transformer.push(line)
             if line is None:
                 return _accumulating(transformer)
 
         #print("transformers clear") #debug
         self.transformer_accumulating = False
         return line
 

IPython/core/inputtransformer.py

@@ -1,534 +1,534 b''
 """Input transformer classes to support IPython special syntax.
 
 This includes the machinery to recognise and transform ``%magic`` commands,
 ``!system`` commands, ``help?`` querying, prompt stripping, and so forth.
 """
 import abc
 import functools
 import re
 from io import StringIO
 
 from IPython.core.splitinput import LineInfo
 from IPython.utils import tokenize2
 from IPython.utils.tokenize2 import generate_tokens, untokenize, TokenError
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # The escape sequences that define the syntax transformations IPython will
 # apply to user input.  These can NOT be just changed here: many regular
 # expressions and other parts of the code may use their hardcoded values, and
 # for all intents and purposes they constitute the 'IPython syntax', so they
 # should be considered fixed.
 
 ESC_SHELL  = '!'     # Send line to underlying system shell
 ESC_SH_CAP = '!!'    # Send line to system shell and capture output
 ESC_HELP   = '?'     # Find information about object
 ESC_HELP2  = '??'    # Find extra-detailed information about object
 ESC_MAGIC  = '%'     # Call magic function
 ESC_MAGIC2 = '%%'    # Call cell-magic function
 ESC_QUOTE  = ','     # Split args on whitespace, quote each as string and call
 ESC_QUOTE2 = ';'     # Quote all args as a single string, call
 ESC_PAREN  = '/'     # Call first argument with rest of line as arguments
 
 ESC_SEQUENCES = [ESC_SHELL, ESC_SH_CAP, ESC_HELP ,\
                  ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,\
                  ESC_QUOTE, ESC_QUOTE2, ESC_PAREN ]
 
 
 class InputTransformer(metaclass=abc.ABCMeta):
     """Abstract base class for line-based input transformers."""
     
     @abc.abstractmethod
     def push(self, line):
         """Send a line of input to the transformer, returning the transformed
         input or None if the transformer is waiting for more input.
         
         Must be overridden by subclasses.
 
         Implementations may raise ``SyntaxError`` if the input is invalid. No
         other exceptions may be raised.
         """
         pass
     
     @abc.abstractmethod
     def reset(self):
         """Return, transformed any lines that the transformer has accumulated,
         and reset its internal state.
         
         Must be overridden by subclasses.
         """
         pass
     
     @classmethod
     def wrap(cls, func):
         """Can be used by subclasses as a decorator, to return a factory that
         will allow instantiation with the decorated object.
         """
         @functools.wraps(func)
         def transformer_factory(**kwargs):
             return cls(func, **kwargs)
         
         return transformer_factory
 
 class StatelessInputTransformer(InputTransformer):
     """Wrapper for a stateless input transformer implemented as a function."""
     def __init__(self, func):
         self.func = func
     
     def __repr__(self):
         return "StatelessInputTransformer(func={0!r})".format(self.func)
     
     def push(self, line):
         """Send a line of input to the transformer, returning the
         transformed input."""
         return self.func(line)
     
     def reset(self):
         """No-op - exists for compatibility."""
         pass
 
 class CoroutineInputTransformer(InputTransformer):
     """Wrapper for an input transformer implemented as a coroutine."""
     def __init__(self, coro, **kwargs):
         # Prime it
         self.coro = coro(**kwargs)
         next(self.coro)
     
     def __repr__(self):
         return "CoroutineInputTransformer(coro={0!r})".format(self.coro)
     
     def push(self, line):
         """Send a line of input to the transformer, returning the
         transformed input or None if the transformer is waiting for more
         input.
         """
         return self.coro.send(line)
     
     def reset(self):
         """Return, transformed any lines that the transformer has
         accumulated, and reset its internal state.
         """
         return self.coro.send(None)
 
 class TokenInputTransformer(InputTransformer):
     """Wrapper for a token-based input transformer.
     
     func should accept a list of tokens (5-tuples, see tokenize docs), and
     return an iterable which can be passed to tokenize.untokenize().
     """
     def __init__(self, func):
         self.func = func
         self.buf = []
         self.reset_tokenizer()
 
     def reset_tokenizer(self):
         it = iter(self.buf)
         self.tokenizer = generate_tokens(it.__next__)
 
     def push(self, line):
         self.buf.append(line + '\n')
         if all(l.isspace() for l in self.buf):
             return self.reset()
 
         tokens = []
         stop_at_NL = False
         try:
             for intok in self.tokenizer:
                 tokens.append(intok)
                 t = intok[0]
                 if t == tokenize2.NEWLINE or (stop_at_NL and t == tokenize2.NL):
                     # Stop before we try to pull a line we don't have yet
                     break
                 elif t == tokenize2.ERRORTOKEN:
                     stop_at_NL = True
         except TokenError:
             # Multi-line statement - stop and try again with the next line
             self.reset_tokenizer()
             return None
         
         return self.output(tokens)
     
     def output(self, tokens):
         self.buf.clear()
         self.reset_tokenizer()
         return untokenize(self.func(tokens)).rstrip('\n')
     
     def reset(self):
         l = ''.join(self.buf)
         self.buf.clear()
         self.reset_tokenizer()
         if l:
             return l.rstrip('\n')
 
 class assemble_python_lines(TokenInputTransformer):
     def __init__(self):
         super(assemble_python_lines, self).__init__(None)
     
     def output(self, tokens):
         return self.reset()
 
 @CoroutineInputTransformer.wrap
 def assemble_logical_lines():
-    """Join lines following explicit line continuations (\)"""
+    r"""Join lines following explicit line continuations (\)"""
     line = ''
     while True:
         line = (yield line)
         if not line or line.isspace():
             continue
         
         parts = []
         while line is not None:
             if line.endswith('\\') and (not has_comment(line)):
                 parts.append(line[:-1])
                 line = (yield None) # Get another line
             else:
                 parts.append(line)
                 break
         
         # Output
         line = ''.join(parts)
 
 # Utilities
 def _make_help_call(target, esc, lspace, next_input=None):
     """Prepares a pinfo(2)/psearch call from a target name and the escape
     (i.e. ? or ??)"""
     method  = 'pinfo2' if esc == '??' \
                 else 'psearch' if '*' in target \
                 else 'pinfo'
     arg = " ".join([method, target])
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = arg.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     if next_input is None:
         return '%sget_ipython().run_line_magic(%r, %r)' % (lspace, t_magic_name, t_magic_arg_s)
     else:
         return '%sget_ipython().set_next_input(%r);get_ipython().run_line_magic(%r, %r)' % \
            (lspace, next_input, t_magic_name, t_magic_arg_s)
     
 # These define the transformations for the different escape characters.
 def _tr_system(line_info):
     "Translate lines escaped with: !"
     cmd = line_info.line.lstrip().lstrip(ESC_SHELL)
     return '%sget_ipython().system(%r)' % (line_info.pre, cmd)
 
 def _tr_system2(line_info):
     "Translate lines escaped with: !!"
     cmd = line_info.line.lstrip()[2:]
     return '%sget_ipython().getoutput(%r)' % (line_info.pre, cmd)
 
 def _tr_help(line_info):
     "Translate lines escaped with: ?/??"
     # A naked help line should just fire the intro help screen
     if not line_info.line[1:]:
         return 'get_ipython().show_usage()'
 
     return _make_help_call(line_info.ifun, line_info.esc, line_info.pre)
 
 def _tr_magic(line_info):
     "Translate lines escaped with: %"
     tpl = '%sget_ipython().run_line_magic(%r, %r)'
     if line_info.line.startswith(ESC_MAGIC2):
         return line_info.line
     cmd = ' '.join([line_info.ifun, line_info.the_rest]).strip()
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     t_magic_name, _, t_magic_arg_s = cmd.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return tpl % (line_info.pre, t_magic_name, t_magic_arg_s)
 
 def _tr_quote(line_info):
     "Translate lines escaped with: ,"
     return '%s%s("%s")' % (line_info.pre, line_info.ifun,
                          '", "'.join(line_info.the_rest.split()) )
 
 def _tr_quote2(line_info):
     "Translate lines escaped with: ;"
     return '%s%s("%s")' % (line_info.pre, line_info.ifun,
                            line_info.the_rest)
 
 def _tr_paren(line_info):
     "Translate lines escaped with: /"
     return '%s%s(%s)' % (line_info.pre, line_info.ifun,
                          ", ".join(line_info.the_rest.split()))
 
 tr = { ESC_SHELL  : _tr_system,
        ESC_SH_CAP : _tr_system2,
        ESC_HELP   : _tr_help,
        ESC_HELP2  : _tr_help,
        ESC_MAGIC  : _tr_magic,
        ESC_QUOTE  : _tr_quote,
        ESC_QUOTE2 : _tr_quote2,
        ESC_PAREN  : _tr_paren }
 
 @StatelessInputTransformer.wrap
 def escaped_commands(line):
     """Transform escaped commands - %magic, !system, ?help + various autocalls.
     """
     if not line or line.isspace():
         return line
     lineinf = LineInfo(line)
     if lineinf.esc not in tr:
         return line
     
     return tr[lineinf.esc](lineinf)
 
 _initial_space_re = re.compile(r'\s*')
 
 _help_end_re = re.compile(r"""(%{0,2}
                               [a-zA-Z_*][\w*]*        # Variable name
                               (\.[a-zA-Z_*][\w*]*)*   # .etc.etc
                               )
                               (\?\??)$                # ? or ??
                               """,
                               re.VERBOSE)
 
 # Extra pseudotokens for multiline strings and data structures
 _MULTILINE_STRING = object()
 _MULTILINE_STRUCTURE = object()
 
 def _line_tokens(line):
     """Helper for has_comment and ends_in_comment_or_string."""
     readline = StringIO(line).readline
     toktypes = set()
     try:
         for t in generate_tokens(readline):
             toktypes.add(t[0])
     except TokenError as e:
         # There are only two cases where a TokenError is raised.
         if 'multi-line string' in e.args[0]:
             toktypes.add(_MULTILINE_STRING)
         else:
             toktypes.add(_MULTILINE_STRUCTURE)
     return toktypes
 
 def has_comment(src):
     """Indicate whether an input line has (i.e. ends in, or is) a comment.
 
     This uses tokenize, so it can distinguish comments from # inside strings.
 
     Parameters
     ----------
     src : string
       A single line input string.
 
     Returns
     -------
     comment : bool
         True if source has a comment.
     """
     return (tokenize2.COMMENT in _line_tokens(src))
 
 def ends_in_comment_or_string(src):
     """Indicates whether or not an input line ends in a comment or within
     a multiline string.
     
     Parameters
     ----------
     src : string
       A single line input string.
 
     Returns
     -------
     comment : bool
         True if source ends in a comment or multiline string.
     """
     toktypes = _line_tokens(src)
     return (tokenize2.COMMENT in toktypes) or (_MULTILINE_STRING in toktypes)
         
 
 @StatelessInputTransformer.wrap
 def help_end(line):
     """Translate lines with ?/?? at the end"""
     m = _help_end_re.search(line)
     if m is None or ends_in_comment_or_string(line):
         return line
     target = m.group(1)
     esc = m.group(3)
     lspace = _initial_space_re.match(line).group(0)
 
     # If we're mid-command, put it back on the next prompt for the user.
     next_input = line.rstrip('?') if line.strip() != m.group(0) else None
 
     return _make_help_call(target, esc, lspace, next_input)
 
 
 @CoroutineInputTransformer.wrap
 def cellmagic(end_on_blank_line=False):
     """Captures & transforms cell magics.
     
     After a cell magic is started, this stores up any lines it gets until it is
     reset (sent None).
     """
     tpl = 'get_ipython().run_cell_magic(%r, %r, %r)'
-    cellmagic_help_re = re.compile('%%\w+\?')
+    cellmagic_help_re = re.compile(r'%%\w+\?')
     line = ''
     while True:
         line = (yield line)
         # consume leading empty lines
         while not line:
             line = (yield line)
         
         if not line.startswith(ESC_MAGIC2):
             # This isn't a cell magic, idle waiting for reset then start over
             while line is not None:
                 line = (yield line)
             continue
         
         if cellmagic_help_re.match(line):
             # This case will be handled by help_end
             continue
         
         first = line
         body = []
         line = (yield None)
         while (line is not None) and \
                                 ((line.strip() != '') or not end_on_blank_line):
             body.append(line)
             line = (yield None)
         
         # Output
         magic_name, _, first = first.partition(' ')
         magic_name = magic_name.lstrip(ESC_MAGIC2)
         line = tpl % (magic_name, first, u'\n'.join(body))
 
 
 def _strip_prompts(prompt_re, initial_re=None, turnoff_re=None):
     """Remove matching input prompts from a block of input.
     
     Parameters
     ----------
     prompt_re : regular expression
         A regular expression matching any input prompt (including continuation)
     initial_re : regular expression, optional
         A regular expression matching only the initial prompt, but not continuation.
         If no initial expression is given, prompt_re will be used everywhere.
         Used mainly for plain Python prompts, where the continuation prompt
         ``...`` is a valid Python expression in Python 3, so shouldn't be stripped.
     
     If initial_re and prompt_re differ,
     only initial_re will be tested against the first line.
     If any prompt is found on the first two lines,
     prompts will be stripped from the rest of the block.
     """
     if initial_re is None:
         initial_re = prompt_re
     line = ''
     while True:
         line = (yield line)
         
         # First line of cell
         if line is None:
             continue
         out, n1 = initial_re.subn('', line, count=1)
         if turnoff_re and not n1:
             if turnoff_re.match(line):
                 # We're in e.g. a cell magic; disable this transformer for
                 # the rest of the cell.
                 while line is not None:
                     line = (yield line)
                 continue
 
         line = (yield out)
         
         if line is None:
             continue
         # check for any prompt on the second line of the cell,
         # because people often copy from just after the first prompt,
         # so we might not see it in the first line.
         out, n2 = prompt_re.subn('', line, count=1)
         line = (yield out)
         
         if n1 or n2:
             # Found a prompt in the first two lines - check for it in
             # the rest of the cell as well.
             while line is not None:
                 line = (yield prompt_re.sub('', line, count=1))
         
         else:
             # Prompts not in input - wait for reset
             while line is not None:
                 line = (yield line)
 
 @CoroutineInputTransformer.wrap
 def classic_prompt():
     """Strip the >>>/... prompts of the Python interactive shell."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(>>>|\.\.\.)( |$)')
     initial_re = re.compile(r'^>>>( |$)')
     # Any %magic/!system is IPython syntax, so we needn't look for >>> prompts
     turnoff_re = re.compile(r'^[%!]')
     return _strip_prompts(prompt_re, initial_re, turnoff_re)
 
 @CoroutineInputTransformer.wrap
 def ipy_prompt():
     """Strip IPython's In [1]:/...: prompts."""
     # FIXME: non-capturing version (?:...) usable?
     prompt_re = re.compile(r'^(In \[\d+\]: |\s*\.{3,}: ?)')
     # Disable prompt stripping inside cell magics
     turnoff_re = re.compile(r'^%%')
     return _strip_prompts(prompt_re, turnoff_re=turnoff_re)
 
 
 @CoroutineInputTransformer.wrap
 def leading_indent():
     """Remove leading indentation.
     
     If the first line starts with a spaces or tabs, the same whitespace will be
     removed from each following line until it is reset.
     """
     space_re = re.compile(r'^[ \t]+')
     line = ''
     while True:
         line = (yield line)
         
         if line is None:
             continue
         
         m = space_re.match(line)
         if m:
             space = m.group(0)
             while line is not None:
                 if line.startswith(space):
                     line = line[len(space):]
                 line = (yield line)
         else:
             # No leading spaces - wait for reset
             while line is not None:
                 line = (yield line)
 
 
 _assign_pat = \
 r'''(?P<lhs>(\s*)
     ([\w\.]+)                # Initial identifier
     (\s*,\s*
         \*?[\w\.]+)*         # Further identifiers for unpacking
     \s*?,?                   # Trailing comma
     )
     \s*=\s*
 '''
 
 assign_system_re = re.compile(r'{}!\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
 assign_system_template = '%s = get_ipython().getoutput(%r)'
 @StatelessInputTransformer.wrap
 def assign_from_system(line):
     """Transform assignment from system commands (e.g. files = !ls)"""
     m = assign_system_re.match(line)
     if m is None:
         return line
     
     return assign_system_template % m.group('lhs', 'cmd')
 
 assign_magic_re = re.compile(r'{}%\s*(?P<cmd>.*)'.format(_assign_pat), re.VERBOSE)
 assign_magic_template = '%s = get_ipython().run_line_magic(%r, %r)'
 @StatelessInputTransformer.wrap
 def assign_from_magic(line):
     """Transform assignment from magic commands (e.g. a = %who_ls)"""
     m = assign_magic_re.match(line)
     if m is None:
         return line
     #Prepare arguments for get_ipython().run_line_magic(magic_name, magic_args)
     m_lhs, m_cmd = m.group('lhs', 'cmd')
     t_magic_name, _, t_magic_arg_s = m_cmd.partition(' ')
     t_magic_name = t_magic_name.lstrip(ESC_MAGIC)
     return assign_magic_template % (m_lhs, t_magic_name, t_magic_arg_s)

IPython/core/magics/config.py

@@ -1,158 +1,158 b''
 """Implementation of configuration-related magic functions.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 The IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import re
 
 # Our own packages
 from IPython.core.error import UsageError
 from IPython.core.magic import Magics, magics_class, line_magic
 from logging import error
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
-reg = re.compile('^\w+\.\w+$')
+reg = re.compile(r'^\w+\.\w+$')
 @magics_class
 class ConfigMagics(Magics):
 
     def __init__(self, shell):
         super(ConfigMagics, self).__init__(shell)
         self.configurables = []
 
     @line_magic
     def config(self, s):
         """configure IPython
 
             %config Class[.trait=value]
 
         This magic exposes most of the IPython config system. Any
         Configurable class should be able to be configured with the simple
         line::
 
             %config Class.trait=value
 
         Where `value` will be resolved in the user's namespace, if it is an
         expression or variable name.
 
         Examples
         --------
 
         To see what classes are available for config, pass no arguments::
 
             In [1]: %config
             Available objects for config:
                 TerminalInteractiveShell
                 HistoryManager
                 PrefilterManager
                 AliasManager
                 IPCompleter
                 DisplayFormatter
 
         To view what is configurable on a given class, just pass the class
         name::
 
             In [2]: %config IPCompleter
             IPCompleter options
             -----------------
             IPCompleter.omit__names=<Enum>
                 Current: 2
                 Choices: (0, 1, 2)
                 Instruct the completer to omit private method names
                 Specifically, when completing on ``object.<tab>``.
                 When 2 [default]: all names that start with '_' will be excluded.
                 When 1: all 'magic' names (``__foo__``) will be excluded.
                 When 0: nothing will be excluded.
             IPCompleter.merge_completions=<CBool>
                 Current: True
                 Whether to merge completion results into a single list
                 If False, only the completion results from the first non-empty
                 completer will be returned.
             IPCompleter.limit_to__all__=<CBool>
                 Current: False
                 Instruct the completer to use __all__ for the completion
                 Specifically, when completing on ``object.<tab>``.
                 When True: only those names in obj.__all__ will be included.
                 When False [default]: the __all__ attribute is ignored
             IPCompleter.greedy=<CBool>
                 Current: False
                 Activate greedy completion
                 This will enable completion on elements of lists, results of
                 function calls, etc., but can be unsafe because the code is
                 actually evaluated on TAB.
 
         but the real use is in setting values::
 
             In [3]: %config IPCompleter.greedy = True
 
         and these values are read from the user_ns if they are variables::
 
             In [4]: feeling_greedy=False
 
             In [5]: %config IPCompleter.greedy = feeling_greedy
 
         """
         from traitlets.config.loader import Config
         # some IPython objects are Configurable, but do not yet have
         # any configurable traits.  Exclude them from the effects of
         # this magic, as their presence is just noise:
         configurables = sorted(set([ c for c in self.shell.configurables
                                      if c.__class__.class_traits(config=True)
                                      ]), key=lambda x: x.__class__.__name__)
         classnames = [ c.__class__.__name__ for c in configurables ]
 
         line = s.strip()
         if not line:
             # print available configurable names
             print("Available objects for config:")
             for name in classnames:
                 print("    ", name)
             return
         elif line in classnames:
             # `%config TerminalInteractiveShell` will print trait info for
             # TerminalInteractiveShell
             c = configurables[classnames.index(line)]
             cls = c.__class__
             help = cls.class_get_help(c)
             # strip leading '--' from cl-args:
             help = re.sub(re.compile(r'^--', re.MULTILINE), '', help)
             print(help)
             return
         elif reg.match(line):
             cls, attr = line.split('.')
             return getattr(configurables[classnames.index(cls)],attr)
         elif '=' not in line:
             msg = "Invalid config statement: %r, "\
                   "should be `Class.trait = value`."
             
             ll = line.lower()
             for classname in classnames:
                 if ll == classname.lower():
                     msg = msg + '\nDid you mean %s (note the case)?' % classname
                     break
 
             raise UsageError( msg % line)
 
         # otherwise, assume we are setting configurables.
         # leave quotes on args when splitting, because we want
         # unquoted args to eval in user_ns
         cfg = Config()
         exec("cfg."+line, locals(), self.shell.user_ns)
 
         for configurable in configurables:
             try:
                 configurable.update_config(cfg)
             except Exception as e:
                 error(e)

IPython/core/splitinput.py

@@ -1,137 +1,137 b''
 # encoding: utf-8
 """
 Simple utility for splitting user input. This is used by both inputsplitter and
 prefilter.
 
 Authors:
 
 * Brian Granger
 * Fernando Perez
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import re
 import sys
 
 from IPython.utils import py3compat
 from IPython.utils.encoding import get_stream_enc
 
 #-----------------------------------------------------------------------------
 # Main function
 #-----------------------------------------------------------------------------
 
 # RegExp for splitting line contents into pre-char//first word-method//rest.
 # For clarity, each group in on one line.
 
 # WARNING: update the regexp if the escapes in interactiveshell are changed, as
 # they are hardwired in.
 
 # Although it's not solely driven by the regex, note that:
 # ,;/% only trigger if they are the first character on the line
 # ! and !! trigger if they are first char(s) *or* follow an indent
 # ? triggers as first or last char.
 
-line_split = re.compile("""
+line_split = re.compile(r"""
              ^(\s*)               # any leading space
              ([,;/%]|!!?|\?\??)?  # escape character or characters
              \s*(%{0,2}[\w\.\*]*)     # function/method, possibly with leading %
                                   # to correctly treat things like '?%magic'
              (.*?$|$)             # rest of line
              """, re.VERBOSE)
 
 
 def split_user_input(line, pattern=None):
     """Split user input into initial whitespace, escape character, function part
     and the rest.
     """
     # We need to ensure that the rest of this routine deals only with unicode
     encoding = get_stream_enc(sys.stdin, 'utf-8')
     line = py3compat.cast_unicode(line, encoding)
 
     if pattern is None:
         pattern = line_split
     match = pattern.match(line)
     if not match:
         # print "match failed for line '%s'" % line
         try:
             ifun, the_rest = line.split(None,1)
         except ValueError:
             # print "split failed for line '%s'" % line
             ifun, the_rest = line, u''
-        pre = re.match('^(\s*)(.*)',line).groups()[0]
+        pre = re.match(r'^(\s*)(.*)',line).groups()[0]
         esc = ""
     else:
         pre, esc, ifun, the_rest = match.groups()
 
     #print 'line:<%s>' % line # dbg
     #print 'pre <%s> ifun <%s> rest <%s>' % (pre,ifun.strip(),the_rest) # dbg
     return pre, esc or '', ifun.strip(), the_rest.lstrip()
 
 
 class LineInfo(object):
     """A single line of input and associated info.
 
     Includes the following as properties:
 
     line
       The original, raw line
 
     continue_prompt
       Is this line a continuation in a sequence of multiline input?
 
     pre
       Any leading whitespace.
 
     esc
       The escape character(s) in pre or the empty string if there isn't one.
       Note that '!!' and '??' are possible values for esc. Otherwise it will
       always be a single character.
 
     ifun
       The 'function part', which is basically the maximal initial sequence
       of valid python identifiers and the '.' character. This is what is
       checked for alias and magic transformations, used for auto-calling,
       etc. In contrast to Python identifiers, it may start with "%" and contain
       "*".
 
     the_rest
       Everything else on the line.
     """
     def __init__(self, line, continue_prompt=False):
         self.line            = line
         self.continue_prompt = continue_prompt
         self.pre, self.esc, self.ifun, self.the_rest = split_user_input(line)
 
         self.pre_char       = self.pre.strip()
         if self.pre_char:
             self.pre_whitespace = '' # No whitespace allowed before esc chars
         else:
             self.pre_whitespace = self.pre
 
     def ofind(self, ip):
         """Do a full, attribute-walking lookup of the ifun in the various
         namespaces for the given IPython InteractiveShell instance.
 
         Return a dict with keys: {found, obj, ospace, ismagic}
 
         Note: can cause state changes because of calling getattr, but should
         only be run if autocall is on and if the line hasn't matched any
         other, less dangerous handlers.
 
         Does cache the results of the call, so can be called multiple times
         without worrying about *further* damaging state.
         """
         return ip._ofind(self.ifun)
 
     def __str__(self):
         return "LineInfo [%s|%s|%s|%s]" %(self.pre, self.esc, self.ifun, self.the_rest)

IPython/utils/path.py

@@ -1,438 +1,438 b''
 # encoding: utf-8
 """
 Utilities for path handling.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 import errno
 import shutil
 import random
 import glob
 from warnings import warn
 
 from IPython.utils.process import system
 from IPython.utils import py3compat
 from IPython.utils.decorators import undoc
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 fs_encoding = sys.getfilesystemencoding()
 
 def _writable_dir(path):
     """Whether `path` is a directory, to which the user has write access."""
     return os.path.isdir(path) and os.access(path, os.W_OK)
 
 if sys.platform == 'win32':
     def _get_long_path_name(path):
         """Get a long path name (expand ~) on Windows using ctypes.
 
         Examples
         --------
 
         >>> get_long_path_name('c:\\docume~1')
         'c:\\\\Documents and Settings'
 
         """
         try:
             import ctypes
         except ImportError:
             raise ImportError('you need to have ctypes installed for this to work')
         _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
         _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
             ctypes.c_uint ]
 
         buf = ctypes.create_unicode_buffer(260)
         rv = _GetLongPathName(path, buf, 260)
         if rv == 0 or rv > 260:
             return path
         else:
             return buf.value
 else:
     def _get_long_path_name(path):
         """Dummy no-op."""
         return path
 
 
 
 def get_long_path_name(path):
     """Expand a path into its long form.
 
     On Windows this expands any ~ in the paths. On other platforms, it is
     a null operation.
     """
     return _get_long_path_name(path)
 
 
 def unquote_filename(name, win32=(sys.platform=='win32')):
     """ On Windows, remove leading and trailing quotes from filenames.
 
     This function has been deprecated and should not be used any more:
     unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
     """
     warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
          "be used anymore", DeprecationWarning, stacklevel=2)
     if win32:
         if name.startswith(("'", '"')) and name.endswith(("'", '"')):
             name = name[1:-1]
     return name
 
 
 def compress_user(path):
     """Reverse of :func:`os.path.expanduser`
     """
     home = os.path.expanduser('~')
     if path.startswith(home):
         path =  "~" + path[len(home):]
     return path
 
 def get_py_filename(name, force_win32=None):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found.
     """
 
     name = os.path.expanduser(name)
     if force_win32 is not None:
         warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
              "since IPython 5.0 and should not be used anymore",
             DeprecationWarning, stacklevel=2)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError('File `%r` not found.' % name)
 
 
 def filefind(filename, path_dirs=None):
     """Find a file by looking through a sequence of paths.
 
     This iterates through a sequence of paths looking for a file and returns
     the full, absolute path of the first occurrence of the file.  If no set of
     path dirs is given, the filename is tested as is, after running through
     :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
 
         filefind('myfile.txt')
 
     will find the file in the current working dir, but::
 
         filefind('~/myfile.txt')
 
     Will find the file in the users home directory.  This function does not
     automatically try any paths, such as the cwd or the user's home directory.
 
     Parameters
     ----------
     filename : str
         The filename to look for.
     path_dirs : str, None or sequence of str
         The sequence of paths to look for the file in.  If None, the filename
         need to be absolute or be in the cwd.  If a string, the string is
         put into a sequence and the searched.  If a sequence, walk through
         each element and join with ``filename``, calling :func:`expandvars`
         and :func:`expanduser` before testing for existence.
 
     Returns
     -------
     Raises :exc:`IOError` or returns absolute path to file.
     """
 
     # If paths are quoted, abspath gets confused, strip them...
     filename = filename.strip('"').strip("'")
     # If the input is an absolute path, just check it exists
     if os.path.isabs(filename) and os.path.isfile(filename):
         return filename
 
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, str):
         path_dirs = (path_dirs,)
 
     for path in path_dirs:
         if path == '.': path = os.getcwd()
         testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
 
     raise IOError("File %r does not exist in any of the search paths: %r" %
                   (filename, path_dirs) )
 
 
 class HomeDirError(Exception):
     pass
 
 
 def get_home_dir(require_writable=False):
     """Return the 'home' directory, as a unicode string.
 
     Uses os.path.expanduser('~'), and checks for writability.
 
     See stdlib docs for how this is determined.
     $HOME is first priority on *ALL* platforms.
 
     Parameters
     ----------
 
     require_writable : bool [default: False]
         if True:
             guarantees the return value is a writable directory, otherwise
             raises HomeDirError
         if False:
             The path is resolved, but it is not guaranteed to exist or be writable.
     """
 
     homedir = os.path.expanduser('~')
     # Next line will make things work even when /home/ is a symlink to
     # /usr/home as it is on FreeBSD, for example
     homedir = os.path.realpath(homedir)
 
     if not _writable_dir(homedir) and os.name == 'nt':
         # expanduser failed, use the registry to get the 'My Documents' folder.
         try:
             try:
                 import winreg as wreg  # Py 3
             except ImportError:
                 import _winreg as wreg  # Py 2
             key = wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
-                "Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
+                r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
             )
             homedir = wreg.QueryValueEx(key,'Personal')[0]
             key.Close()
         except:
             pass
 
     if (not require_writable) or _writable_dir(homedir):
         return py3compat.cast_unicode(homedir, fs_encoding)
     else:
         raise HomeDirError('%s is not a writable dir, '
                 'set $HOME environment variable to override' % homedir)
 
 def get_xdg_dir():
     """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.config if empty OR not set
         xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 def get_xdg_cache_dir():
     """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.cache if empty OR not set
         xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 @undoc
 def get_ipython_dir():
     warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
     from IPython.paths import get_ipython_dir
     return get_ipython_dir()
 
 @undoc
 def get_ipython_cache_dir():
     warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
     from IPython.paths import get_ipython_cache_dir
     return get_ipython_cache_dir()
 
 @undoc
 def get_ipython_package_dir():
     warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
     from IPython.paths import get_ipython_package_dir
     return get_ipython_package_dir()
 
 @undoc
 def get_ipython_module_path(module_str):
     warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
     from IPython.paths import get_ipython_module_path
     return get_ipython_module_path(module_str)
 
 @undoc
 def locate_profile(profile='default'):
     warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
     from IPython.paths import locate_profile
     return locate_profile(profile=profile)
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
 
        In [2]: os.environ['FOO']='test'
 
        In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
     # This is a pretty subtle hack. When expand user is given a UNC path
     # on Windows (\\server\share$\%username%), os.path.expandvars, removes
     # the $ to get (\\server\share\%username%). I think it considered $
     # alone an empty var. But, we need the $ to remains there (it indicates
     # a hidden share).
     if os.name=='nt':
         s = s.replace('$\\', 'IPYTHON_TEMP')
     s = os.path.expandvars(os.path.expanduser(s))
     if os.name=='nt':
         s = s.replace('IPYTHON_TEMP', '$\\')
     return s
 
 
 def unescape_glob(string):
     """Unescape glob pattern in `string`."""
     def unescape(s):
         for pattern in '*[]!?':
             s = s.replace(r'\{0}'.format(pattern), pattern)
         return s
     return '\\'.join(map(unescape, string.split('\\\\')))
 
 
 def shellglob(args):
     """
     Do glob expansion for each element in `args` and return a flattened list.
 
     Unmatched glob pattern will remain as-is in the returned list.
 
     """
     expanded = []
     # Do not unescape backslash in Windows as it is interpreted as
     # path separator:
     unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
     for a in args:
         expanded.extend(glob.glob(a) or [unescape(a)])
     return expanded
 
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         system(cmd)
 
 
 ENOLINK = 1998
 
 def link(src, dst):
     """Hard links ``src`` to ``dst``, returning 0 or errno.
 
     Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
     supported by the operating system.
     """
 
     if not hasattr(os, "link"):
         return ENOLINK
     link_errno = 0
     try:
         os.link(src, dst)
     except OSError as e:
         link_errno = e.errno
     return link_errno
 
 
 def link_or_copy(src, dst):
     """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
 
     Attempts to maintain the semantics of ``shutil.copy``.
 
     Because ``os.link`` does not overwrite files, a unique temporary file
     will be used if the target already exists, then that file will be moved
     into place.
     """
 
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
 
     link_errno = link(src, dst)
     if link_errno == errno.EEXIST:
         if os.stat(src).st_ino == os.stat(dst).st_ino:
             # dst is already a hard link to the correct file, so we don't need
             # to do anything else. If we try to link and rename the file
             # anyway, we get duplicate files - see http://bugs.python.org/issue21876
             return
 
         new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
         try:
             link_or_copy(src, new_dst)
         except:
             try:
                 os.remove(new_dst)
             except OSError:
                 pass
             raise
         os.rename(new_dst, dst)
     elif link_errno != 0:
         # Either link isn't supported, or the filesystem doesn't support
         # linking, or 'src' and 'dst' are on different filesystems.
         shutil.copy(src, dst)
 
 def ensure_dir_exists(path, mode=0o755):
     """ensure that a directory exists
 
     If it doesn't exist, try to create it and protect against a race condition
     if another process is doing the same.
 
     The default permissions are 755, which differ from os.makedirs default of 777.
     """
     if not os.path.exists(path):
         try:
             os.makedirs(path, mode=mode)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
     elif not os.path.isdir(path):
         raise IOError("%r exists but is not a directory" % path)

IPython/utils/text.py

@@ -1,772 +1,772 b''
 # encoding: utf-8
 """
 Utilities for working with strings and text.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.utils.text
    :parts: 3
 """
 
 import os
 import re
 import sys
 import textwrap
 from string import Formatter
 from pathlib import Path
 
 from IPython.utils import py3compat
 
 # datetime.strftime date format for ipython
 if sys.platform == 'win32':
     date_format = "%B %d, %Y"
 else:
     date_format = "%B %-d, %Y"
 
 class LSString(str):
     """String derivative with a special access attributes.
 
     These are normal strings, but with the special attributes:
 
         .l (or .list) : value as list (split on newlines).
         .n (or .nlstr): original value (the string itself).
         .s (or .spstr): value as whitespace-separated string.
         .p (or .paths): list of path objects (requires path.py package)
 
     Any values which require transformations are computed only once and
     cached.
 
     Such strings are very useful to efficiently interact with the shell, which
     typically only understands whitespace-separated options for commands."""
 
     def get_list(self):
         try:
             return self.__list
         except AttributeError:
             self.__list = self.split('\n')
             return self.__list
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = self.replace('\n',' ')
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         return self
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [Path(p) for p in self.split('\n') if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_lsstring(arg):
 #     """ Prettier (non-repr-like) and more informative printer for LSString """
 #     print "LSString (.p, .n, .l, .s available). Value:"
 #     print arg
 #
 #
 # print_lsstring = result_display.when_type(LSString)(print_lsstring)
 
 
 class SList(list):
     """List derivative with a special access attributes.
 
     These are normal lists, but with the special attributes:
 
     * .l (or .list) : value as list (the list itself).
     * .n (or .nlstr): value as a string, joined on newlines.
     * .s (or .spstr): value as a string, joined on spaces.
     * .p (or .paths): list of path objects (requires path.py package)
 
     Any values which require transformations are computed only once and
     cached."""
 
     def get_list(self):
         return self
 
     l = list = property(get_list)
 
     def get_spstr(self):
         try:
             return self.__spstr
         except AttributeError:
             self.__spstr = ' '.join(self)
             return self.__spstr
 
     s = spstr = property(get_spstr)
 
     def get_nlstr(self):
         try:
             return self.__nlstr
         except AttributeError:
             self.__nlstr = '\n'.join(self)
             return self.__nlstr
 
     n = nlstr = property(get_nlstr)
 
     def get_paths(self):
         try:
             return self.__paths
         except AttributeError:
             self.__paths = [Path(p) for p in self if os.path.exists(p)]
             return self.__paths
 
     p = paths = property(get_paths)
 
     def grep(self, pattern, prune = False, field = None):
         """ Return all strings matching 'pattern' (a regex or callable)
 
         This is case-insensitive. If prune is true, return all items
         NOT matching the pattern.
 
         If field is specified, the match must occur in the specified
         whitespace-separated field.
 
         Examples::
 
             a.grep( lambda x: x.startswith('C') )
             a.grep('Cha.*log', prune=1)
             a.grep('chm', field=-1)
         """
 
         def match_target(s):
             if field is None:
                 return s
             parts = s.split()
             try:
                 tgt = parts[field]
                 return tgt
             except IndexError:
                 return ""
 
         if isinstance(pattern, str):
             pred = lambda x : re.search(pattern, x, re.IGNORECASE)
         else:
             pred = pattern
         if not prune:
             return SList([el for el in self if pred(match_target(el))])
         else:
             return SList([el for el in self if not pred(match_target(el))])
 
     def fields(self, *fields):
         """ Collect whitespace-separated fields from string list
 
         Allows quick awk-like usage of string lists.
 
         Example data (in var a, created by 'a = !ls -l')::
 
             -rwxrwxrwx  1 ville None      18 Dec 14  2006 ChangeLog
             drwxrwxrwx+ 6 ville None       0 Oct 24 18:05 IPython
 
         * ``a.fields(0)`` is ``['-rwxrwxrwx', 'drwxrwxrwx+']``
         * ``a.fields(1,0)`` is ``['1 -rwxrwxrwx', '6 drwxrwxrwx+']``
           (note the joining by space).
         * ``a.fields(-1)`` is ``['ChangeLog', 'IPython']``
 
         IndexErrors are ignored.
 
         Without args, fields() just split()'s the strings.
         """
         if len(fields) == 0:
             return [el.split() for el in self]
 
         res = SList()
         for el in [f.split() for f in self]:
             lineparts = []
 
             for fd in fields:
                 try:
                     lineparts.append(el[fd])
                 except IndexError:
                     pass
             if lineparts:
                 res.append(" ".join(lineparts))
 
         return res
 
     def sort(self,field= None,  nums = False):
         """ sort by specified fields (see fields())
 
         Example::
 
             a.sort(1, nums = True)
 
         Sorts a by second field, in numerical order (so that 21 > 3)
 
         """
 
         #decorate, sort, undecorate
         if field is not None:
             dsu = [[SList([line]).fields(field),  line] for line in self]
         else:
             dsu = [[line,  line] for line in self]
         if nums:
             for i in range(len(dsu)):
                 numstr = "".join([ch for ch in dsu[i][0] if ch.isdigit()])
                 try:
                     n = int(numstr)
                 except ValueError:
                     n = 0
                 dsu[i][0] = n
 
 
         dsu.sort()
         return SList([t[1] for t in dsu])
 
 
 # FIXME: We need to reimplement type specific displayhook and then add this
 # back as a custom printer. This should also be moved outside utils into the
 # core.
 
 # def print_slist(arg):
 #     """ Prettier (non-repr-like) and more informative printer for SList """
 #     print "SList (.p, .n, .l, .s, .grep(), .fields(), sort() available):"
 #     if hasattr(arg,  'hideonce') and arg.hideonce:
 #         arg.hideonce = False
 #         return
 #
 #     nlprint(arg)   # This was a nested list printer, now removed.
 #
 # print_slist = result_display.when_type(SList)(print_slist)
 
 
 def indent(instr,nspaces=4, ntabs=0, flatten=False):
     """Indent a string a given number of spaces or tabstops.
 
     indent(str,nspaces=4,ntabs=0) -> indent str by ntabs+nspaces.
 
     Parameters
     ----------
 
     instr : basestring
         The string to be indented.
     nspaces : int (default: 4)
         The number of spaces to be indented.
     ntabs : int (default: 0)
         The number of tabs to be indented.
     flatten : bool (default: False)
         Whether to scrub existing indentation.  If True, all lines will be
         aligned to the same indentation.  If False, existing indentation will
         be strictly increased.
 
     Returns
     -------
 
     str|unicode : string indented by ntabs and nspaces.
 
     """
     if instr is None:
         return
     ind = '\t'*ntabs+' '*nspaces
     if flatten:
         pat = re.compile(r'^\s*', re.MULTILINE)
     else:
         pat = re.compile(r'^', re.MULTILINE)
     outstr = re.sub(pat, ind, instr)
     if outstr.endswith(os.linesep+ind):
         return outstr[:-len(ind)]
     else:
         return outstr
 
 
 def list_strings(arg):
     """Always return a list of strings, given a string or list of strings
     as input.
 
     Examples
     --------
     ::
 
         In [7]: list_strings('A single string')
         Out[7]: ['A single string']
 
         In [8]: list_strings(['A single string in a list'])
         Out[8]: ['A single string in a list']
 
         In [9]: list_strings(['A','list','of','strings'])
         Out[9]: ['A', 'list', 'of', 'strings']
     """
 
     if isinstance(arg, str):
         return [arg]
     else:
         return arg
 
 
 def marquee(txt='',width=78,mark='*'):
     """Return the input string centered in a 'marquee'.
 
     Examples
     --------
     ::
 
         In [16]: marquee('A test',40)
         Out[16]: '**************** A test ****************'
 
         In [17]: marquee('A test',40,'-')
         Out[17]: '---------------- A test ----------------'
 
         In [18]: marquee('A test',40,' ')
         Out[18]: '                 A test                 '
 
     """
     if not txt:
         return (mark*width)[:width]
     nmark = (width-len(txt)-2)//len(mark)//2
     if nmark < 0: nmark =0
     marks = mark*nmark
     return '%s %s %s' % (marks,txt,marks)
 
 
 ini_spaces_re = re.compile(r'^(\s+)')
 
 def num_ini_spaces(strng):
     """Return the number of initial spaces in a string"""
 
     ini_spaces = ini_spaces_re.match(strng)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 
 def format_screen(strng):
     """Format a string for screen printing.
 
     This removes some latex-type format codes."""
     # Paragraph continue
     par_re = re.compile(r'\\$',re.MULTILINE)
     strng = par_re.sub('',strng)
     return strng
 
 
 def dedent(text):
     """Equivalent of textwrap.dedent that ignores unindented first line.
 
     This means it will still dedent strings like:
     '''foo
     is a bar
     '''
 
     For use in wrap_paragraphs.
     """
 
     if text.startswith('\n'):
         # text starts with blank line, don't ignore the first line
         return textwrap.dedent(text)
 
     # split first line
     splits = text.split('\n',1)
     if len(splits) == 1:
         # only one line
         return textwrap.dedent(text)
 
     first, rest = splits
     # dedent everything but the first line
     rest = textwrap.dedent(rest)
     return '\n'.join([first, rest])
 
 
 def wrap_paragraphs(text, ncols=80):
     """Wrap multiple paragraphs to fit a specified width.
 
     This is equivalent to textwrap.wrap, but with support for multiple
     paragraphs, as separated by empty lines.
 
     Returns
     -------
 
     list of complete paragraphs, wrapped to fill `ncols` columns.
     """
     paragraph_re = re.compile(r'\n(\s*\n)+', re.MULTILINE)
     text = dedent(text).strip()
     paragraphs = paragraph_re.split(text)[::2] # every other entry is space
     out_ps = []
     indent_re = re.compile(r'\n\s+', re.MULTILINE)
     for p in paragraphs:
         # presume indentation that survives dedent is meaningful formatting,
         # so don't fill unless text is flush.
         if indent_re.search(p) is None:
             # wrap paragraph
             p = textwrap.fill(p, ncols)
         out_ps.append(p)
     return out_ps
 
 
 def long_substr(data):
     """Return the longest common substring in a list of strings.
     
     Credit: http://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python
     """
     substr = ''
     if len(data) > 1 and len(data[0]) > 0:
         for i in range(len(data[0])):
             for j in range(len(data[0])-i+1):
                 if j > len(substr) and all(data[0][i:i+j] in x for x in data):
                     substr = data[0][i:i+j]
     elif len(data) == 1:
         substr = data[0]
     return substr
 
 
 def strip_email_quotes(text):
     """Strip leading email quotation characters ('>').
 
     Removes any combination of leading '>' interspersed with whitespace that
     appears *identically* in all lines of the input text.
 
     Parameters
     ----------
     text : str
 
     Examples
     --------
 
     Simple uses::
 
         In [2]: strip_email_quotes('> > text')
         Out[2]: 'text'
 
         In [3]: strip_email_quotes('> > text\\n> > more')
         Out[3]: 'text\\nmore'
 
     Note how only the common prefix that appears in all lines is stripped::
 
         In [4]: strip_email_quotes('> > text\\n> > more\\n> more...')
         Out[4]: '> text\\n> more\\nmore...'
 
     So if any line has no quote marks ('>') , then none are stripped from any
     of them ::
     
         In [5]: strip_email_quotes('> > text\\n> > more\\nlast different')
         Out[5]: '> > text\\n> > more\\nlast different'
     """
     lines = text.splitlines()
     matches = set()
     for line in lines:
         prefix = re.match(r'^(\s*>[ >]*)', line)
         if prefix:
             matches.add(prefix.group(1))
         else:
             break
     else:
         prefix = long_substr(list(matches))
         if prefix:
             strip = len(prefix)
             text = '\n'.join([ ln[strip:] for ln in lines])
     return text
 
 def strip_ansi(source):
     """
     Remove ansi escape codes from text.
     
     Parameters
     ----------
     source : str
         Source to remove the ansi from
     """
     return re.sub(r'\033\[(\d|;)+?m', '', source)
 
 
 class EvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Note that this version interprets a : as specifying a format string (as per
     standard string formatting), so if slicing is required, you must explicitly
     create a slice.
     
     This is to be used in templating cases, such as the parallel batch
     script templates, where simple arithmetic on arguments is useful.
 
     Examples
     --------
     ::
 
         In [1]: f = EvalFormatter()
         In [2]: f.format('{n//4}', n=8)
         Out[2]: '2'
 
         In [3]: f.format("{greeting[slice(2,4)]}", greeting="Hello")
         Out[3]: 'll'
     """
     def get_field(self, name, args, kwargs):
         v = eval(name, kwargs)
         return v, name
 
 #XXX: As of Python 3.4, the format string parsing no longer splits on a colon
 # inside [], so EvalFormatter can handle slicing. Once we only support 3.4 and
 # above, it should be possible to remove FullEvalFormatter.
 
 class FullEvalFormatter(Formatter):
     """A String Formatter that allows evaluation of simple expressions.
     
     Any time a format key is not found in the kwargs,
     it will be tried as an expression in the kwargs namespace.
     
     Note that this version allows slicing using [1:2], so you cannot specify
     a format string. Use :class:`EvalFormatter` to permit format strings.
     
     Examples
     --------
     ::
 
         In [1]: f = FullEvalFormatter()
         In [2]: f.format('{n//4}', n=8)
         Out[2]: '2'
 
         In [3]: f.format('{list(range(5))[2:4]}')
         Out[3]: '[2, 3]'
 
         In [4]: f.format('{3*2}')
         Out[4]: '6'
     """
     # copied from Formatter._vformat with minor changes to allow eval
     # and replace the format_spec code with slicing
     def vformat(self, format_string, args, kwargs):
         result = []
         for literal_text, field_name, format_spec, conversion in \
                 self.parse(format_string):
 
             # output the literal text
             if literal_text:
                 result.append(literal_text)
 
             # if there's a field, output it
             if field_name is not None:
                 # this is some markup, find the object and do
                 # the formatting
 
                 if format_spec:
                     # override format spec, to allow slicing:
                     field_name = ':'.join([field_name, format_spec])
 
                 # eval the contents of the field for the object
                 # to be formatted
                 obj = eval(field_name, kwargs)
 
                 # do any conversion on the resulting object
                 obj = self.convert_field(obj, conversion)
 
                 # format the object and append to the result
                 result.append(self.format_field(obj, ''))
 
         return ''.join(py3compat.cast_unicode(s) for s in result)
 
 
 class DollarFormatter(FullEvalFormatter):
     """Formatter allowing Itpl style $foo replacement, for names and attribute
     access only. Standard {foo} replacement also works, and allows full
     evaluation of its arguments.
 
     Examples
     --------
     ::
 
         In [1]: f = DollarFormatter()
         In [2]: f.format('{n//4}', n=8)
         Out[2]: '2'
 
         In [3]: f.format('23 * 76 is $result', result=23*76)
         Out[3]: '23 * 76 is 1748'
 
         In [4]: f.format('$a or {b}', a=1, b=2)
         Out[4]: '1 or 2'
     """
-    _dollar_pattern_ignore_single_quote = re.compile("(.*?)\$(\$?[\w\.]+)(?=([^']*'[^']*')*[^']*$)")
+    _dollar_pattern_ignore_single_quote = re.compile(r"(.*?)\$(\$?[\w\.]+)(?=([^']*'[^']*')*[^']*$)")
     def parse(self, fmt_string):
         for literal_txt, field_name, format_spec, conversion \
                     in Formatter.parse(self, fmt_string):
             
             # Find $foo patterns in the literal text.
             continue_from = 0
             txt = ""
             for m in self._dollar_pattern_ignore_single_quote.finditer(literal_txt):
                 new_txt, new_field = m.group(1,2)
                 # $$foo --> $foo
                 if new_field.startswith("$"):
                     txt += new_txt + new_field
                 else:
                     yield (txt + new_txt, new_field, "", None)
                     txt = ""
                 continue_from = m.end()
             
             # Re-yield the {foo} style pattern
             yield (txt + literal_txt[continue_from:], field_name, format_spec, conversion)
 
 #-----------------------------------------------------------------------------
 # Utils to columnize a list of string
 #-----------------------------------------------------------------------------
 
 def _col_chunks(l, max_rows, row_first=False):
     """Yield successive max_rows-sized column chunks from l."""
     if row_first:
         ncols = (len(l) // max_rows) + (len(l) % max_rows > 0)
         for i in range(ncols):
             yield [l[j] for j in range(i, len(l), ncols)]
     else:
         for i in range(0, len(l), max_rows):
             yield l[i:(i + max_rows)]
 
 
 def _find_optimal(rlist, row_first=False, separator_size=2, displaywidth=80):
     """Calculate optimal info to columnize a list of string"""
     for max_rows in range(1, len(rlist) + 1):
         col_widths = list(map(max, _col_chunks(rlist, max_rows, row_first)))
         sumlength = sum(col_widths)
         ncols = len(col_widths)
         if sumlength + separator_size * (ncols - 1) <= displaywidth:
             break
     return {'num_columns': ncols,
             'optimal_separator_width': (displaywidth - sumlength) // (ncols - 1) if (ncols - 1) else 0,
             'max_rows': max_rows,
             'column_widths': col_widths
             }
 
 
 def _get_or_default(mylist, i, default=None):
     """return list item number, or default if don't exist"""
     if i >= len(mylist):
         return default
     else :
         return mylist[i]
 
 
 def compute_item_matrix(items, row_first=False, empty=None, *args, **kwargs) :
     """Returns a nested list, and info to columnize items
 
     Parameters
     ----------
 
     items
         list of strings to columize
     row_first : (default False)
         Whether to compute columns for a row-first matrix instead of
         column-first (default).
     empty : (default None)
         default value to fill list if needed
     separator_size : int (default=2)
         How much characters will be used as a separation between each columns.
     displaywidth : int (default=80)
         The width of the area onto which the columns should enter
 
     Returns
     -------
 
     strings_matrix
 
         nested list of string, the outer most list contains as many list as
         rows, the innermost lists have each as many element as columns. If the
         total number of elements in `items` does not equal the product of
         rows*columns, the last element of some lists are filled with `None`.
 
     dict_info
         some info to make columnize easier:
 
         num_columns
           number of columns
         max_rows
           maximum number of rows (final number may be less)
         column_widths
           list of with of each columns
         optimal_separator_width
           best separator width between columns
 
     Examples
     --------
     ::
 
         In [1]: l = ['aaa','b','cc','d','eeeee','f','g','h','i','j','k','l']
         In [2]: list, info = compute_item_matrix(l, displaywidth=12)
         In [3]: list
         Out[3]: [['aaa', 'f', 'k'], ['b', 'g', 'l'], ['cc', 'h', None], ['d', 'i', None], ['eeeee', 'j', None]]
         In [4]: ideal = {'num_columns': 3, 'column_widths': [5, 1, 1], 'optimal_separator_width': 2, 'max_rows': 5}
         In [5]: all((info[k] == ideal[k] for k in ideal.keys()))
         Out[5]: True
     """
     info = _find_optimal(list(map(len, items)), row_first, *args, **kwargs)
     nrow, ncol = info['max_rows'], info['num_columns']
     if row_first:
         return ([[_get_or_default(items, r * ncol + c, default=empty) for c in range(ncol)] for r in range(nrow)], info)
     else:
         return ([[_get_or_default(items, c * nrow + r, default=empty) for c in range(ncol)] for r in range(nrow)], info)
 
 
 def columnize(items, row_first=False, separator='  ', displaywidth=80, spread=False):
     """ Transform a list of strings into a single string with columns.
 
     Parameters
     ----------
     items : sequence of strings
         The strings to process.
 
     row_first : (default False)
         Whether to compute columns for a row-first matrix instead of
         column-first (default).
 
     separator : str, optional [default is two spaces]
         The string that separates columns.
 
     displaywidth : int, optional [default is 80]
         Width of the display in number of characters.
 
     Returns
     -------
     The formatted string.
     """
     if not items:
         return '\n'
     matrix, info = compute_item_matrix(items, row_first=row_first, separator_size=len(separator), displaywidth=displaywidth)
     if spread:
         separator = separator.ljust(int(info['optimal_separator_width']))
     fmatrix = [filter(None, x) for x in matrix]
     sjoin = lambda x : separator.join([ y.ljust(w, ' ') for y, w in zip(x, info['column_widths'])])
     return '\n'.join(map(sjoin, fmatrix))+'\n'
 
 
 def get_text_list(list_, last_sep=' and ', sep=", ", wrap_item_with=""):
     """
     Return a string with a natural enumeration of items
 
     >>> get_text_list(['a', 'b', 'c', 'd'])
     'a, b, c and d'
     >>> get_text_list(['a', 'b', 'c'], ' or ')
     'a, b or c'
     >>> get_text_list(['a', 'b', 'c'], ', ')
     'a, b, c'
     >>> get_text_list(['a', 'b'], ' or ')
     'a or b'
     >>> get_text_list(['a'])
     'a'
     >>> get_text_list([])
     ''
     >>> get_text_list(['a', 'b'], wrap_item_with="`")
     '`a` and `b`'
     >>> get_text_list(['a', 'b', 'c', 'd'], " = ", sep=" + ")
     'a + b + c = d'
     """
     if len(list_) == 0:
         return ''
     if wrap_item_with:
         list_ = ['%s%s%s' % (wrap_item_with, item, wrap_item_with) for
                  item in list_]
     if len(list_) == 1:
         return list_[0]
     return '%s%s%s' % (
         sep.join(i for i in list_[:-1]),
         last_sep, list_[-1])

IPython/utils/tokenize2.py

@@ -1,590 +1,590 b''
 """Patched version of standard library tokenize, to deal with various bugs.
 
 Based on Python 3.2 code.
 
 Patches:
 
 - Gareth Rees' patch for Python issue #12691 (untokenizing)
   - Except we don't encode the output of untokenize
   - Python 2 compatible syntax, so that it can be byte-compiled at installation
 - Newlines in comments and blank lines should be either NL or NEWLINE, depending
   on whether they are in a multi-line statement. Filed as Python issue #17061.
 - Export generate_tokens & TokenError
 - u and rb literals are allowed under Python 3.3 and above.
 
 ------------------------------------------------------------------------------
 
 Tokenization help for Python programs.
 
 tokenize(readline) is a generator that breaks a stream of bytes into
 Python tokens.  It decodes the bytes according to PEP-0263 for
 determining source file encoding.
 
 It accepts a readline-like method which is called repeatedly to get the
 next line of input (or b"" for EOF).  It generates 5-tuples with these
 members:
 
     the token type (see token.py)
     the token (a string)
     the starting (row, column) indices of the token (a 2-tuple of ints)
     the ending (row, column) indices of the token (a 2-tuple of ints)
     the original line (string)
 
 It is designed to match the working of the Python tokenizer exactly, except
 that it produces COMMENT tokens for comments and gives type OP for all
 operators.  Additionally, all token lists start with an ENCODING token
 which tells you which encoding was used to decode the bytes stream.
 """
 
 __author__ = 'Ka-Ping Yee <[e-mail]>'
 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                'Michael Foord')
 import builtins
 import re
 import sys
 from token import *
 from codecs import lookup, BOM_UTF8
 import collections
 from io import TextIOWrapper
-cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)")
 
 import token
 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
                            "NL", "untokenize", "ENCODING", "TokenInfo"]
 del token
 
 __all__ += ["generate_tokens", "TokenError"]
 
 COMMENT = N_TOKENS
 tok_name[COMMENT] = 'COMMENT'
 NL = N_TOKENS + 1
 tok_name[NL] = 'NL'
 ENCODING = N_TOKENS + 2
 tok_name[ENCODING] = 'ENCODING'
 N_TOKENS += 3
 
 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
     def __repr__(self):
         annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
         return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
                 self._replace(type=annotated_type))
 
 def group(*choices): return '(' + '|'.join(choices) + ')'
 def any(*choices): return group(*choices) + '*'
 def maybe(*choices): return group(*choices) + '?'
 
 # Note: we use unicode matching for names ("\w") but ascii matching for
 # number literals.
 Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'\w+'
 
 Hexnumber = r'0[xX][0-9a-fA-F]+'
 Binnumber = r'0[bB][01]+'
 Octnumber = r'0[oO][0-7]+'
 Decnumber = r'(?:0+|[1-9][0-9]*)'
 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
 Exponent = r'[eE][-+]?[0-9]+'
 Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
 Expfloat = r'[0-9]+' + Exponent
 Floatnumber = group(Pointfloat, Expfloat)
 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
 
 # Tail end of ' string.
 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
 # Tail end of " string.
 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
 # Tail end of ''' string.
 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
 # Tail end of """ string.
 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
 Triple = group(StringPrefix + "'''", StringPrefix + '"""')
 # Single-line ' or " string.
 String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
                StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
 
 # Because of leftmost-then-longest match semantics, be sure to put the
 # longest operators first (e.g., if = came before ==, == would get
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
                  r"//=?", r"->",
                  r"[+\-*/%&|^=<>]=?",
                  r"~")
 
 Bracket = '[][(){}]'
 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
 Funny = group(Operator, Bracket, Special)
 
 PlainToken = group(Number, Funny, String, Name)
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.
 ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                 group("'", r'\\\r?\n'),
                 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 def _compile(expr):
     return re.compile(expr, re.UNICODE)
 
 tokenprog, pseudoprog, single3prog, double3prog = map(
     _compile, (Token, PseudoToken, Single3, Double3))
 endprogs = {"'": _compile(Single), '"': _compile(Double),
             "'''": single3prog, '"""': double3prog,
             "r'''": single3prog, 'r"""': double3prog,
             "b'''": single3prog, 'b"""': double3prog,
             "R'''": single3prog, 'R"""': double3prog,
             "B'''": single3prog, 'B"""': double3prog,
             "br'''": single3prog, 'br"""': double3prog,
             "bR'''": single3prog, 'bR"""': double3prog,
             "Br'''": single3prog, 'Br"""': double3prog,
             "BR'''": single3prog, 'BR"""': double3prog,
             'r': None, 'R': None, 'b': None, 'B': None}
 
 triple_quoted = {}
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "b'''", 'b"""', "B'''", 'B"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
           "bR'''", 'bR"""', "BR'''", 'BR"""'):
     triple_quoted[t] = t
 single_quoted = {}
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "b'", 'b"', "B'", 'B"',
           "br'", 'br"', "Br'", 'Br"',
           "bR'", 'bR"', "BR'", 'BR"' ):
     single_quoted[t] = t
 
 for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
     _t2 = _prefix+'"""'
     endprogs[_t2] = double3prog
     triple_quoted[_t2] = _t2
     _t1 = _prefix + "'''"
     endprogs[_t1] = single3prog
     triple_quoted[_t1] = _t1
     single_quoted[_prefix+'"'] = _prefix+'"'
     single_quoted[_prefix+"'"] = _prefix+"'"
 del _prefix, _t2, _t1
 endprogs['u'] = None
 endprogs['U'] = None
 
 del _compile
 
 tabsize = 8
 
 class TokenError(Exception): pass
 
 class StopTokenizing(Exception): pass
 
 
 class Untokenizer:
 
     def __init__(self):
         self.tokens = []
         self.prev_row = 1
         self.prev_col = 0
         self.encoding = 'utf-8'
 
     def add_whitespace(self, tok_type, start):
         row, col = start
         assert row >= self.prev_row
         col_offset = col - self.prev_col
         if col_offset > 0:
             self.tokens.append(" " * col_offset)
         elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
             # Line was backslash-continued.
             self.tokens.append(" ")
 
     def untokenize(self, tokens):
         iterable = iter(tokens)
         for t in iterable:
             if len(t) == 2:
                 self.compat(t, iterable)
                 break
             tok_type, token, start, end = t[:4]
             if tok_type == ENCODING:
                 self.encoding = token
                 continue
             self.add_whitespace(tok_type, start)
             self.tokens.append(token)
             self.prev_row, self.prev_col = end
             if tok_type in (NEWLINE, NL):
                 self.prev_row += 1
                 self.prev_col = 0
         return "".join(self.tokens)
 
     def compat(self, token, iterable):
         # This import is here to avoid problems when the itertools
         # module is not built yet and tokenize is imported.
         from itertools import chain
         startline = False
         prevstring = False
         indents = []
         toks_append = self.tokens.append
 
         for tok in chain([token], iterable):
             toknum, tokval = tok[:2]
             if toknum == ENCODING:
                 self.encoding = tokval
                 continue
 
             if toknum in (NAME, NUMBER):
                 tokval += ' '
 
             # Insert a space between two consecutive strings
             if toknum == STRING:
                 if prevstring:
                     tokval = ' ' + tokval
                 prevstring = True
             else:
                 prevstring = False
 
             if toknum == INDENT:
                 indents.append(tokval)
                 continue
             elif toknum == DEDENT:
                 indents.pop()
                 continue
             elif toknum in (NEWLINE, NL):
                 startline = True
             elif startline and indents:
                 toks_append(indents[-1])
                 startline = False
             toks_append(tokval)
 
 
 def untokenize(tokens):
     """
     Convert ``tokens`` (an iterable) back into Python source code. Return
     a bytes object, encoded using the encoding specified by the last
     ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
 
     The result is guaranteed to tokenize back to match the input so that
     the conversion is lossless and round-trips are assured.  The
     guarantee applies only to the token type and token string as the
     spacing between tokens (column positions) may change.
 
     :func:`untokenize` has two modes. If the input tokens are sequences
     of length 2 (``type``, ``string``) then spaces are added as necessary to
     preserve the round-trip property.
 
     If the input tokens are sequences of length 4 or more (``type``,
     ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
     spaces are added so that each token appears in the result at the
     position indicated by ``start`` and ``end``, if possible.
     """
     return Untokenizer().untokenize(tokens)
 
 
 def _get_normal_name(orig_enc):
     """Imitates get_normal_name in tokenizer.c."""
     # Only care about the first 12 characters.
     enc = orig_enc[:12].lower().replace("_", "-")
     if enc == "utf-8" or enc.startswith("utf-8-"):
         return "utf-8"
     if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
        enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
         return "iso-8859-1"
     return orig_enc
 
 def detect_encoding(readline):
     """
     The detect_encoding() function is used to detect the encoding that should
     be used to decode a Python source file.  It requires one argument, readline,
     in the same way as the tokenize() generator.
 
     It will call readline a maximum of twice, and return the encoding used
     (as a string) and a list of any lines (left as bytes) it has read in.
 
     It detects the encoding from the presence of a utf-8 bom or an encoding
     cookie as specified in pep-0263.  If both a bom and a cookie are present,
     but disagree, a SyntaxError will be raised.  If the encoding cookie is an
     invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
     'utf-8-sig' is returned.
 
     If no encoding is specified, then the default of 'utf-8' will be returned.
     """
     bom_found = False
     encoding = None
     default = 'utf-8'
     def read_or_stop():
         try:
             return readline()
         except StopIteration:
             return b''
 
     def find_cookie(line):
         try:
             # Decode as UTF-8. Either the line is an encoding declaration,
             # in which case it should be pure ASCII, or it must be UTF-8
             # per default encoding.
             line_string = line.decode('utf-8')
         except UnicodeDecodeError:
             raise SyntaxError("invalid or missing encoding declaration")
 
         matches = cookie_re.findall(line_string)
         if not matches:
             return None
         encoding = _get_normal_name(matches[0])
         try:
             codec = lookup(encoding)
         except LookupError:
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
         if bom_found:
             if encoding != 'utf-8':
                 # This behaviour mimics the Python interpreter
                 raise SyntaxError('encoding problem: utf-8')
             encoding += '-sig'
         return encoding
 
     first = read_or_stop()
     if first.startswith(BOM_UTF8):
         bom_found = True
         first = first[3:]
         default = 'utf-8-sig'
     if not first:
         return default, []
 
     encoding = find_cookie(first)
     if encoding:
         return encoding, [first]
 
     second = read_or_stop()
     if not second:
         return default, [first]
 
     encoding = find_cookie(second)
     if encoding:
         return encoding, [first, second]
 
     return default, [first, second]
 
 
 def open(filename):
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
     buffer = builtins.open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
     text = TextIOWrapper(buffer, encoding, line_buffering=True)
     text.mode = 'r'
     return text
 
 
 def tokenize(readline):
     """
     The tokenize() generator requires one argument, readline, which
     must be a callable object which provides the same interface as the
     readline() method of built-in file objects.  Each call to the function
     should return one line of input as bytes.  Alternately, readline
     can be a callable function terminating with :class:`StopIteration`::
 
         readline = open(myfile, 'rb').__next__  # Example of alternate readline
 
     The generator produces 5-tuples with these members: the token type; the
     token string; a 2-tuple (srow, scol) of ints specifying the row and
     column where the token begins in the source; a 2-tuple (erow, ecol) of
     ints specifying the row and column where the token ends in the source;
     and the line on which the token was found.  The line passed is the
     logical line; continuation lines are included.
 
     The first token sequence will always be an ENCODING token
     which tells you which encoding was used to decode the bytes stream.
     """
     # This import is here to avoid problems when the itertools module is not
     # built yet and tokenize is imported.
     from itertools import chain, repeat
     encoding, consumed = detect_encoding(readline)
     rl_gen = iter(readline, b"")
     empty = repeat(b"")
     return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
 
 
 def _tokenize(readline, encoding):
     lnum = parenlev = continued = 0
     numchars = '0123456789'
     contstr, needcont = '', 0
     contline = None
     indents = [0]
 
     if encoding is not None:
         if encoding == "utf-8-sig":
             # BOM will already have been stripped.
             encoding = "utf-8"
         yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
     while True:             # loop over lines in stream
         try:
             line = readline()
         except StopIteration:
             line = b''
 
         if encoding is not None:
             line = line.decode(encoding)
         lnum += 1
         pos, max = 0, len(line)
 
         if contstr:                            # continued string
             if not line:
                 raise TokenError("EOF in multi-line string", strstart)
             endmatch = endprog.match(line)
             if endmatch:
                 pos = end = endmatch.end(0)
                 yield TokenInfo(STRING, contstr + line[:end],
                        strstart, (lnum, end), contline + line)
                 contstr, needcont = '', 0
                 contline = None
             elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
                 yield TokenInfo(ERRORTOKEN, contstr + line,
                            strstart, (lnum, len(line)), contline)
                 contstr = ''
                 contline = None
                 continue
             else:
                 contstr = contstr + line
                 contline = contline + line
                 continue
 
         elif parenlev == 0 and not continued:  # new statement
             if not line: break
             column = 0
             while pos < max:                   # measure leading whitespace
                 if line[pos] == ' ':
                     column += 1
                 elif line[pos] == '\t':
                     column = (column//tabsize + 1)*tabsize
                 elif line[pos] == '\f':
                     column = 0
                 else:
                     break
                 pos += 1
             if pos == max:
                 break
 
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
                     nl_pos = pos + len(comment_token)
                     yield TokenInfo(COMMENT, comment_token,
                            (lnum, pos), (lnum, pos + len(comment_token)), line)
                     yield TokenInfo(NEWLINE, line[nl_pos:],
                            (lnum, nl_pos), (lnum, len(line)), line)
                 else:
                     yield TokenInfo(NEWLINE, line[pos:],
                            (lnum, pos), (lnum, len(line)), line)
                 continue
 
             if column > indents[-1]:           # count indents or dedents
                 indents.append(column)
                 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
             while column < indents[-1]:
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",
                         ("<tokenize>", lnum, pos, line))
                 indents = indents[:-1]
                 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
 
         else:                                  # continued statement
             if not line:
                 raise TokenError("EOF in multi-line statement", (lnum, 0))
             continued = 0
 
         while pos < max:
             pseudomatch = pseudoprog.match(line, pos)
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
                 token, initial = line[start:end], line[start]
 
                 if (initial in numchars or                  # ordinary number
                     (initial == '.' and token != '.' and token != '...')):
                     yield TokenInfo(NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
                     yield TokenInfo(NL if parenlev > 0 else NEWLINE,
                            token, spos, epos, line)
                 elif initial == '#':
                     assert not token.endswith("\n")
                     yield TokenInfo(COMMENT, token, spos, epos, line)
                 elif token in triple_quoted:
                     endprog = endprogs[token]
                     endmatch = endprog.match(line, pos)
                     if endmatch:                           # all on one line
                         pos = endmatch.end(0)
                         token = line[start:pos]
                         yield TokenInfo(STRING, token, spos, (lnum, pos), line)
                     else:
                         strstart = (lnum, start)           # multiple lines
                         contstr = line[start:]
                         contline = line
                         break
                 elif initial in single_quoted or \
                     token[:2] in single_quoted or \
                     token[:3] in single_quoted:
                     if token[-1] == '\n':                  # continued string
                         strstart = (lnum, start)
                         endprog = (endprogs[initial] or endprogs[token[1]] or
                                    endprogs[token[2]])
                         contstr, needcont = line[start:], 1
                         contline = line
                         break
                     else:                                  # ordinary string
                         yield TokenInfo(STRING, token, spos, epos, line)
                 elif initial.isidentifier():               # ordinary name
                     yield TokenInfo(NAME, token, spos, epos, line)
                 elif initial == '\\':                      # continued stmt
                     continued = 1
                 else:
                     if initial in '([{':
                         parenlev += 1
                     elif initial in ')]}':
                         parenlev -= 1
                     yield TokenInfo(OP, token, spos, epos, line)
             else:
                 yield TokenInfo(ERRORTOKEN, line[pos],
                            (lnum, pos), (lnum, pos+1), line)
                 pos += 1
 
     for indent in indents[1:]:                 # pop remaining indent levels
         yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
     yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
 
 
 # An undocumented, backwards compatible, API for all the places in the standard
 # library that expect to be able to use tokenize with strings
 def generate_tokens(readline):
     return _tokenize(readline, None)
 
 if __name__ == "__main__":
     # Quick sanity check
     s = b'''def parseline(self, line):
             """Parse the line into a command name and a string containing
             the arguments.  Returns a tuple containing (command, args, line).
             'command' and 'args' may be None if the line couldn't be parsed.
             """
             line = line.strip()
             if not line:
                 return None, None, line
             elif line[0] == '?':
                 line = 'help ' + line[1:]
             elif line[0] == '!':
                 if hasattr(self, 'do_shell'):
                     line = 'shell ' + line[1:]
                 else:
                     return None, None, line
             i, n = 0, len(line)
             while i < n and line[i] in self.identchars: i = i+1
             cmd, arg = line[:i], line[i:].strip()
             return cmd, arg, line
     '''
     for tok in tokenize(iter(s.splitlines()).__next__):
         print(tok)