Implement understanding on __tracebackhide__

__tracebackhide__ is an attribute that when set to a true value mean the
frame need to be skipped/hidden.

Implement this in both ultratb and un ipdb so that by default frames are
hidden, add switches to toggle the behavior.

We reimplement do_up/down but skipping hidden frames when enabled.
This also uses the special value `_ipython_bottom_` to hide all of the
ipython frames.

This also add completion to the debugger

docs/source/whatsnew/pr/skip_tracebackhide.rst

@@ -0,0 +1,11 b''
+The default tracebackmode will now skip frames that are marked with
+``__tracebackhide__ = True`` and show how many traceback frames have been
+skipped. This can be toggled by using :magic:`xmode` with the ``--show`` or
+``--hide`` attribute. It will have no effect on non verbose traceback modes. 
+
+The ipython debugger also now understand ``__tracebackhide__`` as well and will
+skip hidden frames when displaying. Movement up and down the stack will skip the
+hidden frames and will show how many frames were hidden. Internal IPython frames
+are also now hidden by default. The behavior can be changed with the
+``skip_hidden`` command and accepts "yes", "no", "true" and "false" case
+insensitive parameters.

IPython/core/debugger.py

@@ -280,26 +280,31 b' class Pdb(OldPdb):'
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
+        self.skip_hidden = True
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
         self.parser.style = scheme
 
+
+    def hidden_frames(self, stack):
+        """
+        Given an index in the stack return wether it should be skipped.
+
+        This is used in up/down and where to skip frames.
+        """
+        ip_hide = [s[0].f_locals.get("__tracebackhide__", False) for s in stack]
+        ip_start = [i for i, s in enumerate(ip_hide) if s == "__ipython_bottom__"]
+        if ip_start:
+            ip_hide = [h if i > ip_start[0] else True for (i, h) in enumerate(ip_hide)]
+        return ip_hide
+
     def interaction(self, frame, traceback):
         try:
             OldPdb.interaction(self, frame, traceback)
         except KeyboardInterrupt:
-            self.stdout.write('\n' + self.shell.get_exception_only())
-
-    def new_do_up(self, arg):
-        OldPdb.do_up(self, arg)
-    do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
-
-    def new_do_down(self, arg):
-        OldPdb.do_down(self, arg)
-
-    do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
+            self.stdout.write("\n" + self.shell.get_exception_only())
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
@@ -320,6 +325,8 b' class Pdb(OldPdb):'
         return self.do_quit(arg)
 
     def print_stack_trace(self, context=None):
+        Colors = self.color_scheme_table.active_colors
+        ColorsNormal = Colors.Normal
         if context is None:
             context = self.context
         try:
@@ -329,8 +336,21 b' class Pdb(OldPdb):'
         except (TypeError, ValueError) as e:
                 raise ValueError("Context must be a positive integer") from e
         try:
-            for frame_lineno in self.stack:
+            skipped = 0
+            for hidden, frame_lineno in zip(self.hidden_frames(self.stack), self.stack):
+                if hidden and self.skip_hidden:
+                    skipped += 1
+                    continue
+                if skipped:
+                    print(
+                        f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
+                    )
+                    skipped = 0
                 self.print_stack_entry(frame_lineno, context=context)
+            if skipped:
+                print(
+                    f"{Colors.excName}    [... skipping {skipped} hidden frame(s)]{ColorsNormal}\n"
+                )
         except KeyboardInterrupt:
             pass
 
@@ -485,6 +505,16 b' class Pdb(OldPdb):'
         except KeyboardInterrupt:
             pass
 
+    def do_skip_hidden(self, arg):
+        """
+        Change whether or not we should skip frames with the
+        __tracebackhide__ attribute.
+        """
+        if arg.strip().lower() in ("true", "yes"):
+            self.skip_hidden = True
+        elif arg.strip().lower() in ("false", "no"):
+            self.skip_hidden = False
+
     def do_list(self, arg):
         """Print lines of code from the current stack frame
         """
@@ -631,6 +661,109 b' class Pdb(OldPdb):'
 
     do_w = do_where
 
+    def stop_here(self, frame):
+        hidden = False
+        if self.skip_hidden:
+            hidden = frame.f_locals.get("__tracebackhide__", False)
+        if hidden:
+            Colors = self.color_scheme_table.active_colors
+            ColorsNormal = Colors.Normal
+            print(f"{Colors.excName}    [... skipped 1 hidden frame]{ColorsNormal}\n")
+
+        return super().stop_here(frame)
+
+    def do_up(self, arg):
+        """u(p) [count]
+        Move the current frame count (default one) levels up in the
+        stack trace (to an older frame).
+
+        Will skip hidden frames.
+        """
+        ## modified version of upstream that skips
+        # frames with __tracebackide__
+        if self.curindex == 0:
+            self.error("Oldest frame")
+            return
+        try:
+            count = int(arg or 1)
+        except ValueError:
+            self.error("Invalid frame count (%s)" % arg)
+            return
+        skipped = 0
+        if count < 0:
+            _newframe = 0
+        else:
+            _newindex = self.curindex
+            counter = 0
+            hidden_frames = self.hidden_frames(self.stack)
+            for i in range(self.curindex - 1, -1, -1):
+                frame = self.stack[i][0]
+                if hidden_frames[i] and self.skip_hidden:
+                    skipped += 1
+                    continue
+                counter += 1
+                if counter >= count:
+                    break
+            else:
+                # if no break occured.
+                self.error("all frames above hidden")
+                return
+
+            Colors = self.color_scheme_table.active_colors
+            ColorsNormal = Colors.Normal
+            _newframe = i
+        self._select_frame(_newframe)
+        if skipped:
+            print(
+                f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
+            )
+
+    def do_down(self, arg):
+        """d(own) [count]
+        Move the current frame count (default one) levels down in the
+        stack trace (to a newer frame).
+
+        Will skip hidden frames.
+        """
+        if self.curindex + 1 == len(self.stack):
+            self.error("Newest frame")
+            return
+        try:
+            count = int(arg or 1)
+        except ValueError:
+            self.error("Invalid frame count (%s)" % arg)
+            return
+        if count < 0:
+            _newframe = len(self.stack) - 1
+        else:
+            _newindex = self.curindex
+            counter = 0
+            skipped = 0
+            hidden_frames = self.hidden_frames(self.stack)
+            for i in range(self.curindex + 1, len(self.stack)):
+                frame = self.stack[i][0]
+                if hidden_frames[i] and self.skip_hidden:
+                    skipped += 1
+                    continue
+                counter += 1
+                if counter >= count:
+                    break
+            else:
+                self.error("all frames bellow hidden")
+                return
+
+            Colors = self.color_scheme_table.active_colors
+            ColorsNormal = Colors.Normal
+            if skipped:
+                print(
+                    f"{Colors.excName}    [... skipped {skipped} hidden frame(s)]{ColorsNormal}\n"
+                )
+            _newframe = i
+
+        self._select_frame(_newframe)
+
+    do_d = do_down
+    do_u = do_up
 
 class InterruptiblePdb(Pdb):
     """Version of debugger where KeyboardInterrupt exits the debugger altogether."""

IPython/core/interactiveshell.py

@@ -2211,11 +2211,20 b' class InteractiveShell(SingletonConfigurable):'
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
-    def set_custom_completer(self, completer, pos=0):
+    def set_custom_completer(self, completer, pos=0) -> None:
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
-        list where you want the completer to be inserted."""
+        list where you want the completer to be inserted.
+
+        `completer` should have the following signature::
+
+            def completion(self: Completer, text: string) -> List[str]:
+                raise NotImplementedError
+
+        It will be bound to the current Completer instance and pass some text
+        and return a list with current completions to suggest to the user.
+        """
 
         newcomp = types.MethodType(completer, self.Completer)
         self.Completer.custom_matchers.insert(pos,newcomp)
@@ -3310,6 +3319,9 b' class InteractiveShell(SingletonConfigurable):'
         False : successful execution.
         True : an error occurred.
         """
+        # special value to say that anything above is IPython and should be
+        # hidden.
+        __tracebackhide__ = "__ipython_bottom__"
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook, sys.excepthook = sys.excepthook, self.excepthook

IPython/core/magics/basic.py

@@ -364,13 +364,25 b' Currently the magic system has the following functions:""",'
 
         Valid modes: Plain, Context, Verbose, and Minimal.
 
-        If called without arguments, acts as a toggle."""
+        If called without arguments, acts as a toggle.
+
+        When in verbose mode the value --show (and --hide) 
+        will respectively show (or hide) frames with ``__tracebackhide__ =
+        True`` value set.
+        """
 
         def xmode_switch_err(name):
             warn('Error changing %s exception modes.\n%s' %
                  (name,sys.exc_info()[1]))
 
         shell = self.shell
+        if parameter_s.strip() == "--show":
+            shell.InteractiveTB.skip_hidden = False
+            return
+        if parameter_s.strip() == "--hide":
+            shell.InteractiveTB.skip_hidden = True
+            return
+
         new_mode = parameter_s.strip().capitalize()
         try:
             shell.InteractiveTB.set_mode(mode=new_mode)

IPython/core/tests/test_debugger.py

@@ -4,20 +4,24 b''
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+import bdb
+import builtins
+import os
 import signal
+import subprocess
 import sys
 import time
 import warnings
+from subprocess import PIPE, CalledProcessError, check_output
 from tempfile import NamedTemporaryFile
-from subprocess import check_output, CalledProcessError, PIPE
-import subprocess
+from textwrap import dedent
 from unittest.mock import patch
-import builtins
-import bdb
 
 import nose.tools as nt
 
 from IPython.core import debugger
+from IPython.testing import IPYTHON_TESTING_TIMEOUT_SCALE
+from IPython.testing.decorators import skip_win32
 
 #-----------------------------------------------------------------------------
 # Helper classes, from CPython's Pdb test suite
@@ -251,3 +255,69 b' def test_interruptible_core_debugger():'
         # implementation would involve a subprocess, but that adds issues with
         # interrupting subprocesses that are rather complex, so it's simpler
         # just to do it this way.
+
+@skip_win32
+def test_xmode_skip():
+    """that xmode skip frames
+
+    Not as a doctest as pytest does not run doctests.
+    """
+    import pexpect
+    env = os.environ.copy()
+    env["IPY_TEST_SIMPLE_PROMPT"] = "1"
+
+    child = pexpect.spawn(
+        sys.executable, ["-m", "IPython", "--colors=nocolor"], env=env
+    )
+    child.timeout = 15 * IPYTHON_TESTING_TIMEOUT_SCALE
+
+    child.expect("IPython")
+    child.expect("\n")
+    child.expect_exact("In [1]")
+
+    block = dedent(
+        """
+def f():
+    __tracebackhide__ = True
+    g()
+
+def g():
+    raise ValueError
+
+f()
+    """
+    )
+
+    for line in block.splitlines():
+        child.sendline(line)
+        child.expect_exact(line)
+    child.expect_exact("skipping")
+
+    block = dedent(
+        """
+def f():
+    __tracebackhide__ = True
+    g()
+
+def g():
+    from IPython.core.debugger import set_trace
+    set_trace()
+
+f()
+    """
+    )
+
+    for line in block.splitlines():
+        child.sendline(line)
+        child.expect_exact(line)
+
+    child.expect("ipdb>")
+    child.sendline("w")
+    child.expect("hidden")
+    child.expect("ipdb>")
+    child.sendline("skip_hidden false")
+    child.sendline("w")
+    child.expect("__traceba")
+    child.expect("ipdb>")
+
+    child.close()

IPython/core/ultratb.py

@@ -579,12 +579,15 b' class VerboseTB(TBTools):'
         self.check_cache = check_cache
 
         self.debugger_cls = debugger_cls or debugger.Pdb
+        self.skip_hidden = True
 
     def format_record(self, frame_info):
         """Format a single stack frame"""
         Colors = self.Colors  # just a shorthand + quicker name lookup
         ColorsNormal = Colors.Normal  # used a lot
 
+
+
         if isinstance(frame_info, stack_data.RepeatedFrames):
             return '    %s[... skipping similar frames: %s]%s\n' % (
                 Colors.excName, frame_info.description, ColorsNormal)
@@ -699,7 +702,23 b' class VerboseTB(TBTools):'
         head = self.prepare_header(etype, self.long_header)
         records = self.get_records(etb, number_of_lines_of_context, tb_offset)
 
-        frames = list(map(self.format_record, records))
+        frames = []
+        skipped = 0
+        for r in records:
+            if not isinstance(r, stack_data.RepeatedFrames) and self.skip_hidden:
+                if r.frame.f_locals.get("__tracebackhide__", 0):
+                    skipped += 1
+                    continue
+            if skipped:
+                Colors = self.Colors  # just a shorthand + quicker name lookup
+                ColorsNormal = Colors.Normal  # used a lot
+                frames.append(
+                    "    %s[... skipping hidden %s frame]%s\n"
+                    % (Colors.excName, skipped, ColorsNormal)
+                )
+                skipped = 0
+            frames.append(self.format_record(r))
+
 
         formatted_exception = self.format_exception(etype, evalue)
         if records:

IPython/terminal/debugger.py

@@ -41,6 +41,16 b' class TerminalPdb(Pdb):'
                                         global_namespace={},
                                         parent=self.shell,
                                        )
+            # add a completer for all the do_ methods
+            methods_names = [m[3:] for m in dir(self) if m.startswith("do_")]
+
+            def gen_comp(self, text):
+                return [m for m in methods_names if m.startswith(text)]
+            import types
+            newcomp = types.MethodType(gen_comp, compl)
+            compl.custom_matchers.insert(0, newcomp)
+            # end add completer.
+
             self._ptcomp = IPythonPTCompleter(compl)
 
         options = dict(

appveyor.yml

@@ -4,13 +4,6 b' matrix:'
 
 environment:
   matrix:
-   - PYTHON: "C:\\Python36"
-     PYTHON_VERSION: "3.6.x"
-     PYTHON_ARCH: "32"
-
-   - PYTHON: "C:\\Python36-x64"
-     PYTHON_VERSION: "3.6.x"
-     PYTHON_ARCH: "64"
 
    - PYTHON: "C:\\Python37-x64"
      PYTHON_VERSION: "3.7.x"