Merge pull request #10182 from Carreau/reintegrate-jedi

Reintegrate jedi

docs/source/whatsnew/pr/update-completion-api.rst

@@ -0,0 +1,26
+The completer Completion API has seen an overhaul. The old
+``Completer.complete()`` API is waiting deprecation and will soon be replaced
+by ``Completer.completions()``.
+
+This new API is capable of pulling completions from :any:`jedi`, thus allowing
+type inference on non-executed code. If :any:`jedi` is installed completion
+like the following are now becoming possible without code evaluation:
+
+   >>> data = ['Number of users', 123_456]
+   ... data[0].<tab>
+
+IPython is now capable of inferring that `data[0]` is a string, and will
+suggest completions like `.capitalize`. The completion power of IPython will
+increase with new Jedi releases, and a number of bugs and more completions are
+already available on development version of :any:`jedi` if you are curious.
+
+User of the prompt_toolkit interface should also see the type of the item they
+are selecting (method, attribute, module, keyword ...).
+
+The use of Jedi also full fill a number of request and fix a number of bugs
+like case insensitive completion, completion after division operator: See
+:ghpull:`ipython/ipython#10182`.
+
+Extra patches and updates will be needed to the :any:`ipykernel` package for
+this feature to be available for this to be available to other clients like
+Notebook, Lab, Nteract, Hydrogen...

IPython/core/completer.py

@@ -1,12 +1,57
 # encoding: utf-8
-"""Word completion for IPython.
+"""Completion for IPython.
 
 This module started as fork of the rlcompleter module in the Python standard
 library.  The original enhancements made to rlcompleter have been sent
 upstream and were accepted as of Python 2.3,
 
+This module now support a wide variety of completion mechanism both available
+for normal classic Python code, as well as completer for IPython specific
+Syntax like magics.
+
+Experimental
+============
+
+Starting with IPython 6.0, this module can make use of the Jedi library to
+generate completions both using static analysis of the code, and dynamically
+inspecting multiple namespaces. The APIs attached to this new mechanism is
+unstable and will raise unless use in an :any:`provisionalcompleter` context
+manager.
+
+You will find that the following are experimental:
+
+    - :any:`provisionalcompleter`
+    - :any:`IPCompleter.completions`
+    - :any:`Completion`
+    - :any:`rectify_completions`
+
+.. note::
+
+    better name for :any:`rectify_completions` ?
+
+We welcome any feedback on these new API, and we also encourage you to try this
+module in debug mode (start IPython with ``--Completer.debug=True``) in order
+to have extra logging information is :any:`jedi` is crashing, or if current
+IPython completer pending deprecations are returning results not yet handled
+by :any:`jedi`.
+
+Using Jedi for tab completion allow snippets like the following to work without
+having to execute any code:
+
+   >>> myvar = ['hello', 42]
+   ... myvar[1].bi<tab>
+
+Tab completion will be able to infer that ``myvar[1]`` is a real number without
+executing any code unlike the previously available ``IPCompleter.greedy``
+option.
+
+Be sure to update :any:`jedi` to the latest stable version or to try the
+current development version to get better completions.
 """
 
+# skip module docstests
+skip_doctest = True
+
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 #
@@ -26,19 +71,31 import sys
 import unicodedata
 import string
 import warnings
+
+from contextlib import contextmanager
 from importlib import import_module
+from typing import Iterator, List
+from types import SimpleNamespace
 
 from traitlets.config.configurable import Configurable
 from IPython.core.error import TryNext
 from IPython.core.inputsplitter import ESC_MAGIC
 from IPython.core.latex_symbols import latex_symbols, reverse_latex_symbol
 from IPython.utils import generics
-from IPython.utils.decorators import undoc
 from IPython.utils.dir2 import dir2, get_real_method
 from IPython.utils.process import arg_split
 from IPython.utils.py3compat import cast_unicode_py2
 from traitlets import Bool, Enum, observe
 
+try:
+    import jedi
+    import jedi.api.helpers
+    JEDI_INSTALLED = True
+except ImportError:
+    JEDI_INSTALLED = False
+#-----------------------------------------------------------------------------
+# Globals
+#-----------------------------------------------------------------------------
 
 # Public API
 __all__ = ['Completer','IPCompleter']
@@ -49,6 +106,50 else:
     PROTECTABLES = ' ()[]{}?=\\|;:\'#*"^&'
 
 
+_deprecation_readline_sentinel = object()
+
+
+class ProvisionalCompleterWarning(FutureWarning):
+    """
+    Exception raise by an experimental feature in this module.
+
+    Wrap code in :any:`provisionalcompleter` context manager if you
+    are certain you want to use an unstable feature.
+    """
+    pass
+
+warnings.filterwarnings('error', category=ProvisionalCompleterWarning)
+
+@contextmanager
+def provisionalcompleter(action='ignore'):
+    """
+
+
+    This contest manager has to be used in any place where unstable completer
+    behavior and API may be called.
+
+    >>> with provisionalcompleter():
+    ...     completer.do_experimetal_things() # works
+
+    >>> completer.do_experimental_things() # raises.
+
+    .. note:: Unstable
+
+        By using this context manager you agree that the API in use may change
+        without warning, and that you won't complain if they do so.
+
+        You also understand that if the API is not to you liking you should report
+        a bug to explain your use case upstream and improve the API and will loose
+        credibility if you complain after the API is make stable.
+
+        We'll be happy to get your feedback , feature request and improvement on
+        any of the unstable APIs !
+    """
+    with warnings.catch_warnings():
+        warnings.filterwarnings(action, category=ProvisionalCompleterWarning)
+        yield
+
+
 def has_open_quotes(s):
     """Return whether a string has open quotes.
 
@@ -82,7 +183,7 def protect_filename(s):
 
 
 def expand_user(path):
-    """Expand '~'-style usernames in strings.
+    """Expand ``~``-style usernames in strings.
 
     This is similar to :func:`os.path.expanduser`, but it computes and returns
     extra information that will be useful if the input was being used in
@@ -167,8 +268,135 def completions_sorting_key(word):
     return prio1, word, prio2
 
 
-@undoc
-class Bunch(object): pass
+class _FakeJediCompletion:
+    """
+    This is a workaround to communicate to the UI that Jedi has crashed and to
+    report a bug. Will be used only id :any:`IPCompleter.debug` is set to true.
+
+    Added in IPython 6.0 so should likely be removed for 7.0
+
+    """
+
+    def __init__(self, name):
+
+        self.name = name
+        self.complete = name
+        self.type = 'crashed'
+        self.name_with_symbols = name
+
+    def __repr__(self):
+        return '<Fake completion object jedi has crashed>'
+
+
+class Completion:
+    """
+    Completion object used and return by IPython completers.
+
+    .. warning:: Unstable
+        
+        This function is unstable, API may change without warning.
+        It will also raise unless use in proper context manager.
+
+    This act as a middle ground :any:`Completion` object between the
+    :any:`jedi.api.classes.Completion` object and the Prompt Toolkit completion
+    object. While Jedi need a lot of information about evaluator and how the
+    code should be ran/inspected, PromptToolkit (and other frontend) mostly
+    need user facing information.
+
+    - Which range should be replaced replaced by what.
+    - Some metadata (like completion type), or meta informations to displayed to
+      the use user.
+
+    For debugging purpose we can also store the origin of the completion (``jedi``,
+    ``IPython.python_matches``, ``IPython.magics_matches``...).
+    """
+
+    def __init__(self, start: int, end: int, text: str, *, type: str=None, _origin=''):
+        warnings.warn("``Completion`` is a provisional API (as of IPython 6.0). "
+                      "It may change without warnings. "
+                      "Use in corresponding context manager.",
+                      category=ProvisionalCompleterWarning, stacklevel=2)
+
+        self.start = start
+        self.end = end
+        self.text = text
+        self.type = type
+        self._origin = _origin
+
+    def __repr__(self):
+        return '<Completion start=%s end=%s text=%r type=%r>' % (self.start, self.end, self.text, self.type or '?')
+
+    def __eq__(self, other)->Bool:
+        """
+        Equality and hash do not hash the type (as some completer may not be
+        able to infer the type), but are use to (partially) de-duplicate
+        completion.
+
+        Completely de-duplicating completion is a bit tricker that just
+        comparing as it depends on surrounding text, which Completions are not
+        aware of.
+        """
+        return self.start == other.start and \
+            self.end == other.end and \
+            self.text == other.text
+
+    def __hash__(self):
+        return hash((self.start, self.end, self.text))
+
+_IC = Iterator[Completion]
+
+def rectify_completions(text:str, completion:_IC, *, _debug=False)->_IC:
+    """
+    Rectify a set of completion to all have the same ``start`` and ``end``
+
+    .. warning:: Unstable
+
+        This function is unstable, API may change without warning.
+        It will also raise unless use in proper context manager.
+
+    Parameters
+    ----------
+    text: str
+        text that should be completed.
+    completion: Iterator[Completion]
+        iterator over the completions to rectify
+
+
+    :any:`jedi.api.classes.Completion` s returned by Jedi may not have the same start and end, though
+    the Jupyter Protocol requires them to behave like so. This will readjust
+    the completion to have the same ``start`` and ``end` by padding both
+    extremities with surrounding text.
+
+    During stabilisation should support a ``_debug`` option to log which
+    completion are return by the IPython completer and not found in Jedi in
+    order to make upstream bug report.
+    """
+    warnings.warn("`rectify_completions` is a provisional API (as of IPython 6.0). "
+                 "It may change without warnings. "
+                 "Use in corresponding context manager.",
+                  category=ProvisionalCompleterWarning, stacklevel=2)
+
+    completions = list(completion)
+    if not completions:
+        return
+    starts = (c.start for c in completions)
+    ends = (c.end for c in completions)
+
+    new_start = min(starts)
+    new_end = max(ends)
+
+    seen_jedi = set()
+    seen_python_matches = set()
+    for c in completions:
+        new_text = text[new_start:c.start] + c.text + text[c.end:new_end]
+        if c._origin == 'jedi':
+            seen_jedi.add(new_text)
+        elif c._origin == 'IPCompleter.python_matches':
+            seen_python_matches.add(new_text)
+        yield Completion(new_start, new_end, new_text, type=c.type, _origin=c._origin)
+    diff = seen_python_matches.difference(seen_jedi)
+    if diff and _debug:
+        print('IPython.python matches have extras:', diff)
 
 
 if sys.platform == 'win32':
@@ -189,7 +417,7 class CompletionSplitter(object):
     entire line.
 
     What characters are used as splitting delimiters can be controlled by
-    setting the `delims` attribute (this is a property that internally
+    setting the ``delims`` attribute (this is a property that internally
     automatically builds the necessary regular expression)"""
 
     # Private interface
@@ -230,6 +458,7 class CompletionSplitter(object):
         return self._delim_re.split(l)[-1]
 
 
+
 class Completer(Configurable):
 
     greedy = Bool(False,
@@ -240,7 +469,16 class Completer(Configurable):
         but can be unsafe because the code is actually evaluated on TAB.
         """
     ).tag(config=True)
-    
+
+    use_jedi = Bool(default_value=JEDI_INSTALLED,
+                    help="Experimental: Use Jedi to generate autocompletions. "
+                    "Default to True if jedi is installed").tag(config=True)
+
+    debug = Bool(default_value=False,
+                 help='Enable debug for the Completer. Mostly print extra '
+                      'information for experimental jedi integration.')\
+                      .tag(config=True)
+
 
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
@@ -254,20 +492,15 class Completer(Configurable):
         An optional second namespace can be given.  This allows the completer
         to handle cases where both the local and global scopes need to be
         distinguished.
-
-        Completer instances should be used as the completion mechanism of
-        readline via the set_completer() call:
-
-        readline.set_completer(Completer(my_namespace).complete)
         """
 
         # Don't bind to namespace quite yet, but flag whether the user wants a
         # specific namespace or to use __main__.__dict__. This will allow us
         # to bind to __main__.__dict__ at completion time, not now.
         if namespace is None:
-            self.use_main_ns = 1
+            self.use_main_ns = True
         else:
-            self.use_main_ns = 0
+            self.use_main_ns = False
             self.namespace = namespace
 
         # The global namespace, if given, can be bound directly
@@ -354,13 +587,15 class Completer(Configurable):
 
         if self.limit_to__all__ and hasattr(obj, '__all__'):
             words = get__all__entries(obj)
-        else: 
+        else:
             words = dir2(obj)
 
         try:
             words = generics.complete_object(obj, words)
         except TryNext:
             pass
+        except AssertionError:
+            raise
         except Exception:
             # Silence errors from completion function
             #raise # dbg
@@ -376,7 +611,7 def get__all__entries(obj):
         words = getattr(obj, '__all__')
     except:
         return []
-    
+ 
     return [cast_unicode_py2(w) for w in words if isinstance(w, str)]
 
 
@@ -431,6 +666,73 def match_dict_keys(keys, prefix, delims):
     return quote, token_start, matched
 
 
+def cursor_to_position(text:int, line:int, column:int)->int:
+    """
+
+    Convert the (line,column) position of the cursor in text to an offset in a
+    string.
+
+    Parameter
+    ---------
+
+    text : str
+        The text in which to calculate the cursor offset
+    line : int
+        Line of the cursor; 0-indexed
+    column : int
+        Column of the cursor 0-indexed
+
+    Return
+    ------
+        Position of the cursor in ``text``, 0-indexed.
+
+    See Also
+    --------
+    position_to_cursor: reciprocal of this function
+
+    """
+    lines = text.split('\n')
+    assert line <= len(lines), '{} <= {}'.format(str(line), str(len(lines)))
+
+    return sum(len(l) + 1 for l in lines[:line]) + column
+
+def position_to_cursor(text:str, offset:int)->(int, int):
+    """
+    Convert the position of the cursor in text (0 indexed) to a line
+    number(0-indexed) and a column number (0-indexed) pair
+
+    Position should be a valid position in ``text``.
+
+    Parameter
+    ---------
+
+    text : str
+        The text in which to calculate the cursor offset
+    offset : int
+        Position of the cursor in ``text``, 0-indexed.
+
+    Return
+    ------
+    (line, column) : (int, int)
+        Line of the cursor; 0-indexed, column of the cursor 0-indexed
+
+
+    See Also
+    --------
+    cursor_to_position : reciprocal of this function
+
+
+    """
+
+    assert 0 < offset <= len(text) , "0 < %s <= %s" % (offset , len(text))
+
+    before = text[:offset]
+    blines = before.split('\n')  # ! splitnes trim trailing \n
+    line = before.count('\n')
+    col = len(blines[-1])
+    return line, col
+
+
 def _safe_isinstance(obj, module, class_name):
     """Checks if obj is an instance of module.class_name if loaded
     """
@@ -441,7 +743,7 def _safe_isinstance(obj, module, class_name):
 def back_unicode_name_matches(text):
     u"""Match unicode characters back to unicode name
     
-    This does  ☃ -> \\snowman
+    This does  ``☃`` -> ``\\snowman``
 
     Note that snowman is not a valid python3 combining character but will be expanded.
     Though it will not recombine back to the snowman character by the completion machinery.
@@ -468,10 +770,10 def back_unicode_name_matches(text):
         pass
     return u'', ()
 
-def back_latex_name_matches(text):
-    u"""Match latex characters back to unicode name
+def back_latex_name_matches(text:str):
+    """Match latex characters back to unicode name
     
-    This does  ->\\sqrt
+    This does ``\\ℵ`` -> ``\\aleph``
 
     Used on Python 3 only.
     """
@@ -536,7 +838,7 class IPCompleter(Completer):
         
         When True: only those names in obj.__all__ will be included.
         
-        When False [default]: the __all__ attribute is ignored 
+        When False [default]: the __all__ attribute is ignored
         """,
     ).tag(config=True)
 
@@ -548,33 +850,36 class IPCompleter(Completer):
             UserWarning)
 
     def __init__(self, shell=None, namespace=None, global_namespace=None,
-                 use_readline=False, config=None, **kwargs):
+                 use_readline=_deprecation_readline_sentinel, config=None, **kwargs):
         """IPCompleter() -> completer
 
-        Return a completer object suitable for use by the readline library
-        via readline.set_completer().
+        Return a completer object.
 
-        Inputs:
-
-        - shell: a pointer to the ipython shell itself.  This is needed
-          because this completer knows about magic functions, and those can
-          only be accessed via the ipython instance.
+        Parameters
+        ----------
 
-        - namespace: an optional dict where completions are performed.
+        shell
+            a pointer to the ipython shell itself.  This is needed
+            because this completer knows about magic functions, and those can
+            only be accessed via the ipython instance.
 
-        - global_namespace: secondary optional dict for completions, to
-          handle cases (such as IPython embedded inside functions) where
-          both Python scopes are visible.
+        namespace : dict, optional
+            an optional dict where completions are performed.
 
+        global_namespace : dict, optional
+            secondary optional dict for completions, to
+            handle cases (such as IPython embedded inside functions) where
+            both Python scopes are visible.
+       
         use_readline : bool, optional
-          DEPRECATED, ignored.
+            DEPRECATED, ignored since IPython 6.0, will have no effects
         """
 
         self.magic_escape = ESC_MAGIC
         self.splitter = CompletionSplitter()
 
-        if use_readline:
-            warnings.warn('The use_readline parameter is deprecated and ignored since IPython 6.0.',
+        if use_readline is not _deprecation_readline_sentinel:
+            warnings.warn('The `use_readline` parameter is deprecated and ignored since IPython 6.0.',
                           DeprecationWarning, stacklevel=2)
 
         # _greedy_changed() depends on splitter and readline being defined:
@@ -733,6 +1038,55 class IPCompleter(Completer):
             comp += [ pre+m for m in line_magics if m.startswith(bare_text)]
         return [cast_unicode_py2(c) for c in comp]
 
+    def _jedi_matches(self, cursor_column:int, cursor_line:int, text:str):
+        """
+
+        Return a list of :any:`jedi.api.Completions` object from a ``text`` and
+        cursor position.
+
+        Parameters
+        ----------
+        cursor_column : int
+            column position of the cursor in ``text``, 0-indexed.
+        cursor_line : int
+            line position of the cursor in ``text``, 0-indexed
+        text : str
+            text to complete
+
+        Debugging
+        ---------
+
+        If ``IPCompleter.debug`` is ``True`` may return a :any:`_FakeJediCompletion`
+        object containing a string with the Jedi debug information attached.
+        """
+        namespaces = [self.namespace]
+        if self.global_namespace is not None:
+            namespaces.append(self.global_namespace)
+
+        # cursor_pos is an it, jedi wants line and column
+        offset = cursor_to_position(text, cursor_line, cursor_column)
+        if offset:
+            pre = text[offset-1]
+        completion_filter = lambda x:x
+        if pre == '.':
+            if self.omit__names == 2:
+                completion_filter = lambda c:not c.name.startswith('_')
+            elif self.omit__names == 1:
+                completion_filter = lambda c:not (c.name.startswith('__') and c.name.endswith('__'))
+            elif self.omit__names == 0:
+                completion_filter = lambda x:x
+            else:
+                raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
+
+        interpreter = jedi.Interpreter(
+            text, namespaces, column=cursor_column, line=cursor_line + 1)
+        try:
+            return filter(completion_filter, interpreter.completions())
+        except Exception as e:
+            if self.debug:
+                return [_FakeJediCompletion('Opps Jedi has crash please report a bug with the following:\n"""\n%s\ns"""' % (e))]
+            else:
+                return []
 
     def python_matches(self, text):
         """Match attributes or global python names"""
@@ -1006,12 +1360,12 class IPCompleter(Completer):
         return [leading + k + suf for k in matches]
 
     def unicode_name_matches(self, text):
-        u"""Match Latex-like syntax for unicode characters base 
+        u"""Match Latex-like syntax for unicode characters base
         on the name of the character.
         
-        This does  \\GREEK SMALL LETTER ETA -> η
+        This does  ``\\GREEK SMALL LETTER ETA`` -> ``η``
 
-        Works only on valid python 3 identifier, or on combining characters that 
+        Works only on valid python 3 identifier, or on combining characters that
         will combine to form a valid identifier.
         
         Used on Python 3 only.
@@ -1029,13 +1383,11 class IPCompleter(Completer):
         return u'', []
 
 
-
-
     def latex_matches(self, text):
         u"""Match Latex syntax for unicode characters.
         
-        This does both \\alp -> \\alpha and \\alpha -> α
-        
+        This does both ``\\alp`` -> ``\\alpha`` and ``\\alpha`` -> ``α``
+
         Used on Python 3 only.
         """
         slashpos = text.rfind('\\')
@@ -1062,7 +1414,7 class IPCompleter(Completer):
 
         # Create a little structure to pass all the relevant information about
         # the current completion to any custom completer.
-        event = Bunch()
+        event = SimpleNamespace()
         event.line = line
         event.symbol = text
         cmd = line.split(None,1)[0]
@@ -1094,6 +1446,94 class IPCompleter(Completer):
 
         return None
 
+    def completions(self, text: str, offset: int)->Iterator[Completion]:
+        """
+        Returns an iterator over the possible completions
+
+        .. warning:: Unstable
+            
+            This function is unstable, API may change without warning.
+            It will also raise unless use in proper context manager.
+
+        Parameters
+        ----------
+        
+        text:str
+            Full text of the current input, multi line string.
+        offset:int
+            Integer representing the position of the cursor in ``text``. Offset
+            is 0-based indexed.
+
+        Yields
+        ------
+            :any:`Completion` object
+
+
+        The cursor on a text can either be seen as being "in between"
+        characters or "On" a character depending on the interface visible to
+        the user. For consistency the cursor being on "in between" characters X
+        and Y is equivalent to the cursor being "on" character Y, that is to say
+        the character the cursor is on is considered as being after the cursor.
+
+        Combining characters may span more that one position in the
+        text.
+
+
+        .. note::
+
+            If ``IPCompleter.debug`` is :any:`True` will yield a ``--jedi/ipython--``
+            fake Completion token to distinguish completion returned by Jedi
+            and usual IPython completion.
+
+        """
+        warnings.warn("_complete is a provisional API (as of IPython 6.0). "
+                      "It may change without warnings. "
+                      "Use in corresponding context manager.",
+                      category=ProvisionalCompleterWarning, stacklevel=2)
+
+        # Possible Improvements / Known limitation
+        ##########################################
+        # Completions may be identical even if they have different ranges and
+        # text. For example:
+        # >>> a=1
+        # >>> a.<tab>
+        # May returns:
+        #  - `a.real` from 0 to 2
+        #  - `.real` from 1 to 2
+        # the current code does not (yet) check for such equivalence
+        seen = set()
+        for c in self._completions(text, offset):
+            if c and (c in seen):
+                continue
+            yield c
+            seen.add(c)
+
+    def _completions(self, full_text: str, offset: int)->Iterator[Completion]:
+        before = full_text[:offset]
+        cursor_line, cursor_column = position_to_cursor(full_text, offset)
+
+        matched_text, matches, matches_origin, jedi_matches = self._complete(
+            full_text=full_text, cursor_line=cursor_line, cursor_pos=cursor_column)
+
+        for jm in jedi_matches:
+            delta = len(jm.name_with_symbols) - len(jm.complete)
+            yield Completion(start=offset - delta, end=offset, text=jm.name_with_symbols, type=jm.type, _origin='jedi')
+
+
+        start_offset = before.rfind(matched_text)
+
+        # TODO:
+        # Supress this, right now just for debug.
+        if jedi_matches and matches and self.debug:
+            yield Completion(start=start_offset, end=offset, text='--jedi/ipython--', _origin='debug')
+
+        # I'm unsure if this is always true, so let's assert and see if it
+        # crash
+        assert before.endswith(matched_text)
+        for m, t in zip(matches, matches_origin):
+            yield Completion(start=start_offset, end=offset, text=m, _origin=t)
+
+
     def complete(self, text=None, line_buffer=None, cursor_pos=None):
         """Find completions for the given text and line context.
 
@@ -1123,7 +1563,38 class IPCompleter(Completer):
 
         matches : list
           A list of completion matches.
+
+
+        .. note::
+
+            This API is likely to be deprecated and replaced by
+            :any:`IPCompleter.completions` in the future.
+
+
+        """
+        warnings.warn('`Completer.complete` is pending deprecation since '
+                'IPython 6.0 and will be replaced by `Completer.completions`.',
+                      PendingDeprecationWarning)
+        # potential todo, FOLD the 3rd throw away argument of _complete
+        # into the first 2 one.
+        return self._complete(line_buffer=line_buffer, cursor_pos=cursor_pos, text=text, cursor_line=0)[:2]
+
+    def _complete(self, *, cursor_line, cursor_pos, line_buffer=None, text=None,
+                  full_text=None, return_jedi_results=True) -> (str, List[str], List[object]):
+        """
+
+        Like complete but can also returns raw jedi completions as well as the
+        origin of the completion text. This could (and should) be made much
+        cleaner but that will be simpler once we drop the old (and stateful)
+        :any:`complete` API.
+
+
+        With current provisional API, cursor_pos act both (depending on the
+        caller) as the offset in the ``text`` or ``line_buffer``, or as the
+        ``column`` when passing multiline strings this could/should be renamed
+        but would add extra noise.
         """
+
         # if the cursor position isn't given, the only sane assumption we can
         # make is that it's at the end of the line (the common case)
         if cursor_pos is None:
@@ -1132,20 +1603,23 class IPCompleter(Completer):
         if self.use_main_ns:
             self.namespace = __main__.__dict__
 
+        # if text is either None or an empty string, rely on the line buffer
+        if (not line_buffer) and full_text:
+            line_buffer = full_text.split('\n')[cursor_line]
+        if not text:
+            text = self.splitter.split_line(line_buffer, cursor_pos)
+
         base_text = text if not line_buffer else line_buffer[:cursor_pos]
         latex_text, latex_matches = self.latex_matches(base_text)
         if latex_matches:
-            return latex_text, latex_matches
+            return latex_text, latex_matches, ['latex_matches']*len(latex_matches), ()
         name_text = ''
         name_matches = []
         for meth in (self.unicode_name_matches, back_latex_name_matches, back_unicode_name_matches):
             name_text, name_matches = meth(base_text)
             if name_text:
-                return name_text, name_matches
+                return name_text, name_matches, [meth.__qualname__]*len(name_matches), {}
         
-        # if text is either None or an empty string, rely on the line buffer
-        if not text:
-            text = self.splitter.split_line(line_buffer, cursor_pos)
 
         # If no line buffer is given, assume the input text is all there was
         if line_buffer is None:
@@ -1155,33 +1629,55 class IPCompleter(Completer):
         self.text_until_cursor = self.line_buffer[:cursor_pos]
 
         # Start with a clean slate of completions
-        self.matches[:] = []
+        matches = []
         custom_res = self.dispatch_custom_completer(text)
+        # FIXME: we should extend our api to return a dict with completions for
+        # different types of objects.  The rlcomplete() method could then
+        # simply collapse the dict into a list for readline, but we'd have
+        # richer completion semantics in other evironments.
+        completions = ()
+        if self.use_jedi and return_jedi_results:
+            if not full_text:
+                full_text = line_buffer
+            completions = self._jedi_matches(
+                cursor_pos, cursor_line, full_text)
         if custom_res is not None:
             # did custom completers produce something?
-            self.matches = custom_res
+            matches = [(m, 'custom') for m in custom_res]
         else:
             # Extend the list of completions with the results of each
             # matcher, so we return results to the user from all
             # namespaces.
             if self.merge_completions:
-                self.matches = []
+                matches = []
                 for matcher in self.matchers:
                     try:
-                        self.matches.extend(matcher(text))
+                        matches.extend([(m, matcher.__qualname__)
+                                        for m in matcher(text)])
                     except:
                         # Show the ugly traceback if the matcher causes an
                         # exception, but do NOT crash the kernel!
                         sys.excepthook(*sys.exc_info())
             else:
                 for matcher in self.matchers:
-                    self.matches = matcher(text)
-                    if self.matches:
+                    matches = [(m, matcher.__qualname__)
+                               for m in matcher(text)]
+                    if matches:
                         break
-        # FIXME: we should extend our api to return a dict with completions for
-        # different types of objects.  The rlcomplete() method could then
-        # simply collapse the dict into a list for readline, but we'd have
-        # richer completion semantics in other evironments.
-        self.matches = sorted(set(self.matches), key=completions_sorting_key)
+        seen = set()
+        filtered_matches = set()
+        for m in matches:
+            t, c = m
+            if t not in seen:
+                filtered_matches.add(m)
+                seen.add(t)
+
+        filtered_matches = sorted(
+            set(filtered_matches), key=lambda x: completions_sorting_key(x[0]))
+
+        matches = [m[0] for m in filtered_matches]
+        origins = [m[1] for m in filtered_matches]
+
+        self.matches = matches
 
-        return text, self.matches
+        return text, matches, origins, completions

IPython/core/interactiveshell.py

@@ -11,7 +11,6
 #-----------------------------------------------------------------------------
 
 
-import __future__
 import abc
 import ast
 import atexit

IPython/core/tests/test_completer.py

@@ -20,6 +20,9 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from IPython.utils.generics import complete_object
 from IPython.testing import decorators as dec
 
+from IPython.core.completer import Completion, provisionalcompleter
+from nose.tools import assert_in, assert_not_in
+
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
@@ -269,23 +272,65 def test_local_file_completions():
         nt.assert_true(comp.issubset(set(c)))
 
 
+def test_jedi():
+    """
+    A couple of issue we had with Jedi
+    """
+    ip = get_ipython()
+
+    def _test_complete(reason, s, comp, start=None, end=None):
+        l = len(s)
+        start = start if start is not None else l
+        end = end if end is not None else l
+        with provisionalcompleter():
+            completions = set(ip.Completer.completions(s, l))
+            assert_in(Completion(start, end, comp), completions, reason)
+
+    def _test_not_complete(reason, s, comp):
+        l = len(s)
+        with provisionalcompleter():
+            completions = set(ip.Completer.completions(s, l))
+            assert_not_in(Completion(l, l, comp), completions, reason)
+
+    import jedi
+    jedi_version = tuple(int(i) for i in jedi.__version__.split('.')[:3])
+    if jedi_version > (0,10):
+        yield _test_complete, 'jedi >0.9 should complete and not crash', 'a=1;a.', 'real'
+    yield _test_complete, 'can infer first argument', 'a=(1,"foo");a[0].', 'real'
+    yield _test_complete, 'can infer second argument', 'a=(1,"foo");a[1].', 'capitalize'
+    yield _test_complete, 'cover duplicate completions', 'im', 'import', 0, 2
+
+    yield _test_not_complete, 'does not mix types', 'a=(1,"foo");a[0].', 'capitalize'
+
+
 def test_greedy_completions():
+    """
+    Test the capability of the Greedy completer. 
+
+    Most of the test here do not really show off the greedy completer, for proof
+    each of the text bellow now pass with Jedi. The greedy completer is capable of more. 
+
+    See the :any:`test_dict_key_completion_contexts`
+
+    """
     ip = get_ipython()
     ip.ex('a=list(range(5))')
     _,c = ip.complete('.',line='a[0].')
     nt.assert_false('.real' in c,
                     "Shouldn't have completed on a[0]: %s"%c)
-    with greedy_completion():
-        def _(line, cursor_pos, expect, message):
+    with greedy_completion(), provisionalcompleter():
+        def _(line, cursor_pos, expect, message, completion):
             _,c = ip.complete('.', line=line, cursor_pos=cursor_pos)
+            with provisionalcompleter():
+                completions = ip.Completer.completions(line, cursor_pos)
             nt.assert_in(expect, c, message%c)
+            nt.assert_in(completion, completions)
 
-        yield _, 'a[0].', 5, 'a[0].real', "Should have completed on a[0].: %s"
-        yield _, 'a[0].r', 6, 'a[0].real', "Should have completed on a[0].r: %s"
-        
-        if sys.version_info > (3,4):
-            yield _, 'a[0].from_', 10, 'a[0].from_bytes', "Should have completed on a[0].from_: %s"
+        yield _, 'a[0].', 5, 'a[0].real', "Should have completed on a[0].: %s", Completion(5,5, 'real')
+        yield _, 'a[0].r', 6, 'a[0].real', "Should have completed on a[0].r: %s", Completion(5,6, 'real')
 
+        if sys.version_info > (3, 4):
+            yield _, 'a[0].from_', 10, 'a[0].from_bytes', "Should have completed on a[0].from_: %s", Completion(5, 10, 'from_bytes')
 
 
 def test_omit__names():
@@ -298,27 +343,58 def test_omit__names():
     cfg = Config()
     cfg.IPCompleter.omit__names = 0
     c.update_config(cfg)
-    s,matches = c.complete('ip.')
-    nt.assert_in('ip.__str__', matches)
-    nt.assert_in('ip._hidden_attr', matches)
+    with provisionalcompleter():
+        s,matches = c.complete('ip.')
+        completions = set(c.completions('ip.', 3))
+
+        nt.assert_in('ip.__str__', matches)
+        nt.assert_in(Completion(3, 3, '__str__'), completions)
+        
+        nt.assert_in('ip._hidden_attr', matches)
+        nt.assert_in(Completion(3,3, "_hidden_attr"), completions)
+
+
     cfg = Config()
     cfg.IPCompleter.omit__names = 1
     c.update_config(cfg)
-    s,matches = c.complete('ip.')
-    nt.assert_not_in('ip.__str__', matches)
-    nt.assert_in('ip._hidden_attr', matches)
+    with provisionalcompleter():
+        s,matches = c.complete('ip.')
+        completions = set(c.completions('ip.', 3))
+
+        nt.assert_not_in('ip.__str__', matches)
+        nt.assert_not_in(Completion(3,3,'__str__'), completions)
+
+        # nt.assert_in('ip._hidden_attr', matches)
+        nt.assert_in(Completion(3,3, "_hidden_attr"), completions)
+
     cfg = Config()
     cfg.IPCompleter.omit__names = 2
     c.update_config(cfg)
-    s,matches = c.complete('ip.')
-    nt.assert_not_in('ip.__str__', matches)
-    nt.assert_not_in('ip._hidden_attr', matches)
-    s,matches = c.complete('ip._x.')
-    nt.assert_in('ip._x.keys', matches)
+    with provisionalcompleter():
+        s,matches = c.complete('ip.')
+        completions = set(c.completions('ip.', 3))
+
+        nt.assert_not_in('ip.__str__', matches)
+        nt.assert_not_in(Completion(3,3,'__str__'), completions)
+
+        nt.assert_not_in('ip._hidden_attr', matches)
+        nt.assert_not_in(Completion(3,3, "_hidden_attr"), completions)
+
+    with provisionalcompleter():
+        s,matches = c.complete('ip._x.')
+        completions = set(c.completions('ip._x.', 6))
+
+        nt.assert_in('ip._x.keys', matches)
+        nt.assert_in(Completion(6,6, "keys"), completions)
+
     del ip._hidden_attr
+    del ip._x
 
 
 def test_limit_to__all__False_ok():
+    """
+    Limit to all is deprecated, once we remove it this test can go away. 
+    """
     ip = get_ipython()
     c = ip.Completer
     ip.ex('class D: x=24')

IPython/terminal/ptutils.py

@@ -10,13 +10,17 not to be used outside IPython.
 import unicodedata
 from wcwidth import wcwidth
 
-from IPython.core.completer import IPCompleter
+from IPython.core.completer import IPCompleter, provisionalcompleter, rectify_completions, cursor_to_position
 from prompt_toolkit.completion import Completer, Completion
 from prompt_toolkit.layout.lexers import Lexer
 from prompt_toolkit.layout.lexers import PygmentsLexer
 
 import pygments.lexers as pygments_lexers
 
+_completion_sentinel = object()
+
+
+
 
 class IPythonPTCompleter(Completer):
     """Adaptor to provide IPython completions to prompt_toolkit"""
@@ -39,35 +43,44 class IPythonPTCompleter(Completer):
     def get_completions(self, document, complete_event):
         if not document.current_line.strip():
             return
-
         # Some bits of our completion system may print stuff (e.g. if a module
         # is imported). This context manager ensures that doesn't interfere with
         # the prompt.
-        with self.patch_stdout():
-            used, matches = self.ipy_completer.complete(
-                                line_buffer=document.current_line,
-                                cursor_pos=document.cursor_position_col
-            )
-        start_pos = -len(used)
-        for m in matches:
-            if not m:
+
+        with self.patch_stdout(), provisionalcompleter():
+            body = document.text
+            cursor_row = document.cursor_position_row
+            cursor_col = document.cursor_position_col
+            cursor_position = document.cursor_position
+            offset = cursor_to_position(body, cursor_row, cursor_col)
+            yield from self._get_completions(body, offset, cursor_position, self.ipy_completer)
+
+    @staticmethod
+    def _get_completions(body, offset, cursor_position, ipyc):
+        """
+        Private equivalent of get_completions() use only for unit_testing.
+        """
+        debug = getattr(ipyc, 'debug', False)
+        completions = rectify_completions(
+            body, ipyc.completions(body, offset), _debug=debug)
+        for c in completions:
+            if not c.text:
                 # Guard against completion machinery giving us an empty string.
                 continue
-
-            m = unicodedata.normalize('NFC', m)
-
+            text = unicodedata.normalize('NFC', c.text)
             # When the first character of the completion has a zero length,
             # then it's probably a decomposed unicode character. E.g. caused by
             # the "\dot" completion. Try to compose again with the previous
             # character.
-            if wcwidth(m[0]) == 0:
-                if document.cursor_position + start_pos > 0:
-                    char_before = document.text[document.cursor_position + start_pos - 1]
-                    m = unicodedata.normalize('NFC', char_before + m)
+            if wcwidth(text[0]) == 0:
+                if cursor_position + c.start > 0:
+                    char_before = body[c.start - 1]
+                    fixed_text = unicodedata.normalize(
+                        'NFC', char_before + text)
 
                     # Yield the modified completion instead, if this worked.
-                    if wcwidth(m[0:1]) == 1:
-                        yield Completion(m, start_position=start_pos - 1)
+                    if wcwidth(text[0:1]) == 1:
+                        yield Completion(fixed_text, start_position=c.start - offset - 1)
                         continue
 
             # TODO: Use Jedi to determine meta_text
@@ -75,7 +88,7 class IPythonPTCompleter(Completer):
             # meta_text = ''
             # yield Completion(m, start_position=start_pos,
             #                  display_meta=meta_text)
-            yield Completion(m, start_position=start_pos)
+            yield Completion(c.text, start_position=c.start - offset, display_meta=c.type)
 
 class IPythonPTLexer(Lexer):
     """

IPython/testing/iptest.py

@@ -58,11 +58,24 warnings.filterwarnings('error', message='.*disable_gui.*', category=Deprecation
 
 warnings.filterwarnings('error', message='.*ExceptionColors global is deprecated.*', category=DeprecationWarning, module='.*')
 
+# Jedi older versions
+warnings.filterwarnings(
+    'error', message='.*elementwise != comparison failed and.*', category=FutureWarning, module='.*')
+
 if version_info < (6,):
     # nose.tools renames all things from `camelCase` to `snake_case` which raise an
     # warning with the runner they also import from standard import library. (as of Dec 2015)
     # Ignore, let's revisit that in a couple of years for IPython 6.
-    warnings.filterwarnings('ignore', message='.*Please use assertEqual instead', category=Warning, module='IPython.*')
+    warnings.filterwarnings(
+        'ignore', message='.*Please use assertEqual instead', category=Warning, module='IPython.*')
+
+if version_info < (7,):
+    warnings.filterwarnings('ignore', message='.*Completer.complete.*',
+                            category=PendingDeprecationWarning, module='.*')
+else:
+    warnings.warn(
+        'Completer.complete was pending deprecation and should be changed to Deprecated', FutureWarning)
+
 
 
 # ------------------------------------------------------------------------------

IPython/testing/iptestcontroller.py

@@ -11,12 +11,9 test suite.
 
 
 import argparse
-import json
 import multiprocessing.pool
 import os
 import stat
-import re
-import requests
 import shutil
 import signal
 import sys
@@ -25,13 +22,11 import time
 
 from .iptest import (
     have, test_group_names as py_test_group_names, test_sections, StreamCapturer,
-    test_for,
 )
 from IPython.utils.path import compress_user
 from IPython.utils.py3compat import bytes_to_str
 from IPython.utils.sysinfo import get_sys_info
 from IPython.utils.tempdir import TemporaryDirectory
-from IPython.utils.text import strip_ansi
 
 try:
     # Python >= 3.3
@@ -50,8 +45,6 except ImportError:
         if p.poll() is None:
             raise TimeoutExpired
 
-NOTEBOOK_SHUTDOWN_TIMEOUT = 10
-
 class TestController(object):
     """Run tests in a subprocess
     """

docs/source/conf.py

@@ -228,6 +228,8 intersphinx_mapping = {'python': ('https://docs.python.org/3/', None),
                        'jupyterclient': ('https://jupyter-client.readthedocs.io/en/latest/', None),
                        'ipyparallel': ('https://ipyparallel.readthedocs.io/en/latest/', None),
                        'jupyter': ('https://jupyter.readthedocs.io/en/latest/', None),
+                       'jedi': ('https://jedi.readthedocs.io/en/latest/', None),
+                       'traitlets': ('https://traitlets.readthedocs.io/en/latest/', None),
                       }
 
 # Options for LaTeX output

docs/source/index.rst

@@ -104,6 +104,7 repository <http://github.com/ipython/ipython>`_.
 
 
 .. only:: html
+
    * :ref:`genindex`
    * :ref:`modindex`
    * :ref:`search`

setup.py

@@ -201,6 +201,8 extras_require = dict(
 
 install_requires = [
     'setuptools>=18.5',
+    'jedi',
+    'typing',
     'decorator',
     'pickleshare',
     'simplegeneric>0.8',