Backport PR #12809: Allow passing a custom CachingCompiler to the interactive shell

IPython/core/compilerop.py

@@ -99,7 +99,7 b' class CachingCompiler(codeop.Compile):'
         Arguments are exactly the same as ast.parse (in the standard library),
         and are passed to the built-in compile function."""
         return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
-    
+
     def reset_compiler_flags(self):
         """Reset compiler flags to default state."""
         # This value is copied from codeop.Compile.__init__, so if that ever
@@ -112,25 +112,53 b' class CachingCompiler(codeop.Compile):'
         """
         return self.flags
 
-    def cache(self, code, number=0):
+    def get_code_name(self, raw_code, transformed_code, number):
+        """Compute filename given the code, and the cell number.
+
+        Parameters
+        ----------
+        raw_code : str
+          The raw cell code.
+        transformed_code : str
+          The executable Python source code to cache and compile.
+        number : int
+          A number which forms part of the code's name. Used for the execution
+          counter.
+
+        Returns
+        -------
+        The computed filename.
+        """
+        return code_name(transformed_code, number)
+
+    def cache(self, transformed_code, number=0, raw_code=None):
         """Make a name for a block of code, and cache the code.
 
         Parameters
         ----------
-        code : str
-          The Python source code to cache.
+        transformed_code : str
+          The executable Python source code to cache and compile.
         number : int
           A number which forms part of the code's name. Used for the execution
           counter.
+        raw_code : str
+          The raw code before transformation, if None, set to `transformed_code`.
 
         Returns
         -------
         The name of the cached code (as a string). Pass this as the filename
         argument to compilation, so that tracebacks are correctly hooked up.
         """
-        name = code_name(code, number)
-        entry = (len(code), time.time(),
-                 [line+'\n' for line in code.splitlines()], name)
+        if raw_code is None:
+            raw_code = transformed_code
+
+        name = self.get_code_name(raw_code, transformed_code, number)
+        entry = (
+            len(transformed_code),
+            time.time(),
+            [line + "\n" for line in transformed_code.splitlines()],
+            name,
+        )
         linecache.cache[name] = entry
         linecache._ipython_cache[name] = entry
         return name
@@ -146,7 +174,7 b' class CachingCompiler(codeop.Compile):'
             yield
         finally:
             # turn off only the bits we turned on so that something like
-            # __future__ that set flags stays. 
+            # __future__ that set flags stays.
             self.flags &= ~turn_on_bits
 
 

IPython/core/interactiveshell.py

@@ -340,7 +340,7 b' class InteractiveShell(SingletonConfigurable):'
     """An enhanced, interactive shell for Python."""
 
     _instance = None
-    
+
     ast_transformers = List([], help=
         """
         A list of ast.NodeTransformer subclass instances, which will be applied
@@ -407,7 +407,7 b' class InteractiveShell(SingletonConfigurable):'
         Enable magic commands to be called without the leading %.
         """
     ).tag(config=True)
-    
+
     banner1 = Unicode(default_banner,
         help="""The part of the banner to be printed before the profile"""
     ).tag(config=True)
@@ -443,7 +443,8 b' class InteractiveShell(SingletonConfigurable):'
     display_formatter = Instance(DisplayFormatter, allow_none=True)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
-    
+    compiler_class = Type(CachingCompiler)
+
     sphinxify_docstring = Bool(False, help=
         """
         Enables rich html representation of docstrings. (This requires the
@@ -534,7 +535,7 b' class InteractiveShell(SingletonConfigurable):'
     ).tag(config=True)
 
     # deprecated prompt traits:
-    
+
     prompt_in1 = Unicode('In [\\#]: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
@@ -547,14 +548,14 b' class InteractiveShell(SingletonConfigurable):'
     prompts_pad_left = Bool(True,
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
-    
+
     @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
     def _prompt_trait_changed(self, change):
         name = change['name']
         warn("InteractiveShell.{name} is deprecated since IPython 4.0"
              " and ignored since 5.0, set TerminalInteractiveShell.prompts"
              " object directly.".format(name=name))
-        
+
         # protect against weird cases where self.config may not exist:
 
     show_rewritten_input = Bool(True,
@@ -638,7 +639,7 b' class InteractiveShell(SingletonConfigurable):'
         self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
-        
+
         # Check if we're in a virtualenv, and set up sys.path.
         self.init_virtualenv()
 
@@ -748,7 +749,7 b' class InteractiveShell(SingletonConfigurable):'
         self.more = False
 
         # command compiler
-        self.compile = CachingCompiler()
+        self.compile = self.compiler_class()
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
@@ -975,7 +976,7 b' class InteractiveShell(SingletonConfigurable):'
     #-------------------------------------------------------------------------
     # Things related to the banner
     #-------------------------------------------------------------------------
-    
+
     @property
     def banner(self):
         banner = self.banner1
@@ -989,7 +990,7 b' class InteractiveShell(SingletonConfigurable):'
         if banner is None:
             banner = self.banner
         sys.stdout.write(banner)
-    
+
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
@@ -1006,10 +1007,10 b' class InteractiveShell(SingletonConfigurable):'
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
-        
+
         if self.display_page:
             self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
-    
+
     def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
                  _warn_deprecated=True):
         """set_hook(name,hook) -> sets an internal IPython hook.
@@ -1073,7 +1074,7 b' class InteractiveShell(SingletonConfigurable):'
         warn("ip.register_post_execute is deprecated, use "
              "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
         self.events.register('post_run_cell', func)
-    
+
     def _clear_warning_registry(self):
         # clear the warning registry, so that different code blocks with
         # overlapping line number ranges don't cause spurious suppression of
@@ -1115,12 +1116,12 b' class InteractiveShell(SingletonConfigurable):'
         else:
             main_mod.__dict__.clear()
             main_mod.__name__ = modname
-        
+
         main_mod.__file__ = filename
         # It seems pydoc (and perhaps others) needs any module instance to
         # implement a __nonzero__ method
         main_mod.__nonzero__ = lambda : True
-        
+
         return main_mod
 
     def clear_main_mod_cache(self):
@@ -1273,7 +1274,7 b' class InteractiveShell(SingletonConfigurable):'
                          'user_local':self.user_ns,
                          'builtin':builtin_mod.__dict__
                          }
-    
+
     @property
     def user_global_ns(self):
         return self.user_module.__dict__
@@ -1306,17 +1307,17 b' class InteractiveShell(SingletonConfigurable):'
             user_ns.setdefault("__name__", "__main__")
             user_module = DummyMod()
             user_module.__dict__ = user_ns
-            
+
         if user_module is None:
             user_module = types.ModuleType("__main__",
                 doc="Automatically created module for IPython interactive environment")
-        
+
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         user_module.__dict__.setdefault('__builtin__', builtin_mod)
         user_module.__dict__.setdefault('__builtins__', builtin_mod)
-        
+
         if user_ns is None:
             user_ns = user_module.__dict__
 
@@ -1372,7 +1373,7 b' class InteractiveShell(SingletonConfigurable):'
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = {}
-        
+
         # make global variables for user access to the histories
         ns['_ih'] = self.history_manager.input_hist_parsed
         ns['_oh'] = self.history_manager.output_hist
@@ -1385,7 +1386,7 b' class InteractiveShell(SingletonConfigurable):'
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
-        
+
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
 
@@ -1399,7 +1400,7 b' class InteractiveShell(SingletonConfigurable):'
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
-    
+
     @property
     def all_ns_refs(self):
         """Get a list of references to all the namespace dictionaries in which
@@ -1425,7 +1426,7 b' class InteractiveShell(SingletonConfigurable):'
         # Reset last execution result
         self.last_execution_succeeded = True
         self.last_execution_result = None
-        
+
         # Flush cached output items
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
@@ -1490,7 +1491,7 b' class InteractiveShell(SingletonConfigurable):'
             raise ValueError("Refusing to delete %s" % varname)
 
         ns_refs = self.all_ns_refs
-        
+
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
@@ -1765,8 +1766,14 b' class InteractiveShell(SingletonConfigurable):'
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
-                pmethod(info.obj, oname, formatter, info, 
-                        enable_html_pager=self.enable_html_pager, **kw)
+                pmethod(
+                    info.obj,
+                    oname,
+                    formatter,
+                    info,
+                    enable_html_pager=self.enable_html_pager,
+                    **kw
+                )
             else:
                 pmethod(info.obj, oname)
         else:
@@ -1890,7 +1897,7 b' class InteractiveShell(SingletonConfigurable):'
             print('Exception type :', etype)
             print('Exception value:', value)
             print('Traceback      :', tb)
-        
+
         def validate_stb(stb):
             """validate structured traceback return type
             
@@ -1919,7 +1926,7 b' class InteractiveShell(SingletonConfigurable):'
         else:
             def wrapped(self,etype,value,tb,tb_offset=None):
                 """wrap CustomTB handler, to protect IPython from user code
-                
+
                 This makes it harder (but not impossible) for custom exception
                 handlers to crash IPython.
                 """
@@ -1968,10 +1975,10 b' class InteractiveShell(SingletonConfigurable):'
 
     def _get_exc_info(self, exc_tuple=None):
         """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
-        
+
         Ensures sys.last_type,value,traceback hold the exc_info we found,
         from whichever source.
-        
+
         raises ValueError if none of these contain any information
         """
         if exc_tuple is None:
@@ -1983,10 +1990,10 b' class InteractiveShell(SingletonConfigurable):'
             if hasattr(sys, 'last_type'):
                 etype, value, tb = sys.last_type, sys.last_value, \
                                    sys.last_traceback
-        
+
         if etype is None:
             raise ValueError("No exception to find")
-        
+
         # Now store the exception info in sys.last_type etc.
         # WARNING: these variables are somewhat deprecated and not
         # necessarily safe to use in a threaded environment, but tools
@@ -1995,16 +2002,16 b' class InteractiveShell(SingletonConfigurable):'
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = tb
-        
+
         return etype, value, tb
-    
+
     def show_usage_error(self, exc):
         """Show a short message for UsageErrors
-        
+
         These are special exceptions that shouldn't show a traceback.
         """
         print("UsageError: %s" % exc, file=sys.stderr)
-    
+
     def get_exception_only(self, exc_tuple=None):
         """
         Return as a string (ending with a newline) the exception that
@@ -2118,7 +2125,7 b' class InteractiveShell(SingletonConfigurable):'
 
     def init_readline(self):
         """DEPRECATED
-        
+
         Moved to terminal subclass, here only to simplify the init logic."""
         # Set a number of methods that depend on readline to be no-op
         warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
@@ -2285,12 +2292,13 b' class InteractiveShell(SingletonConfigurable):'
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.run_line_magic('colors', self.colors)
-    
+
     # Defined here so that it's included in the documentation
     @functools.wraps(magic.MagicsManager.register_function)
     def register_magic_function(self, func, magic_kind='line', magic_name=None):
-        self.magics_manager.register_function(func, 
-                                  magic_kind=magic_kind, magic_name=magic_name)
+        self.magics_manager.register_function(
+            func, magic_kind=magic_kind, magic_name=magic_name
+        )
 
     def run_line_magic(self, magic_name, line, _stack_depth=1):
         """Execute the given line magic.
@@ -2525,7 +2533,7 b' class InteractiveShell(SingletonConfigurable):'
                 ec = 130
             if ec > 128:
                 ec = -(ec - 128)
-        
+
         # We explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.  Note the semantics
@@ -2588,7 +2596,7 b' class InteractiveShell(SingletonConfigurable):'
     def init_payload(self):
         self.payload_manager = PayloadManager(parent=self)
         self.configurables.append(self.payload_manager)
-    
+
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
@@ -2628,13 +2636,13 b' class InteractiveShell(SingletonConfigurable):'
 
     def _user_obj_error(self):
         """return simple exception dict
-        
+
         for use in user_expressions
         """
-        
+
         etype, evalue, tb = self._get_exc_info()
         stb = self.InteractiveTB.get_exception_only(etype, evalue)
-        
+
         exc_info = {
             u'status' : 'error',
             u'traceback' : stb,
@@ -2643,13 +2651,13 b' class InteractiveShell(SingletonConfigurable):'
         }
 
         return exc_info
-    
+
     def _format_user_obj(self, obj):
         """format a user object to display dict
-        
+
         for use in user_expressions
         """
-        
+
         data, md = self.display_formatter.format(obj)
         value = {
             'status' : 'ok',
@@ -2657,7 +2665,7 b' class InteractiveShell(SingletonConfigurable):'
             'metadata' : md,
         }
         return value
-    
+
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
@@ -2676,7 +2684,7 b' class InteractiveShell(SingletonConfigurable):'
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
-        
+
         for key, expr in expressions.items():
             try:
                 value = self._format_user_obj(eval(expr, global_ns, user_ns))
@@ -3092,12 +3100,14 b' class InteractiveShell(SingletonConfigurable):'
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler
-        compiler = self.compile if shell_futures else CachingCompiler()
+        compiler = self.compile if shell_futures else self.compiler_class()
 
         _run_async = False
 
         with self.builtin_trap:
-            cell_name = self.compile.cache(cell, self.execution_count)
+            cell_name = self.compile.cache(
+                cell, self.execution_count, raw_code=raw_cell
+            )
 
             with self.display_trap:
                 # Compile to bytecode
@@ -3203,13 +3213,13 b' class InteractiveShell(SingletonConfigurable):'
 
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
-        
+
         Parameters
         ----------
         node : ast.Node
           The root node to be transformed. Typically called with the ast.Module
           produced by parsing user input.
-        
+
         Returns
         -------
         An ast.Node corresponding to the node it was called with. Note that it
@@ -3256,7 +3266,7 b' class InteractiveShell(SingletonConfigurable):'
           Experimental value: 'async' Will try to run top level interactive
           async/await code in default runner, this will not respect the
           interactivity setting and will only run the last node if it is an
-          expression. 
+          expression.
 
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
@@ -3477,17 +3487,17 b' class InteractiveShell(SingletonConfigurable):'
 
     def enable_gui(self, gui=None):
         raise NotImplementedError('Implement enable_gui in a subclass')
-    
+
     def enable_matplotlib(self, gui=None):
         """Enable interactive matplotlib and inline figure support.
-        
+
         This takes the following steps:
-        
+
         1. select the appropriate eventloop and matplotlib backend
         2. set up matplotlib for interactive use with that backend
         3. configure formatters for inline figure display
         4. enable the selected gui eventloop
-        
+
         Parameters
         ----------
         gui : optional, string
@@ -3501,7 +3511,7 b' class InteractiveShell(SingletonConfigurable):'
         """
         from IPython.core import pylabtools as pt
         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
-    
+
         if gui != 'inline':
             # If we have our first gui selection, store it
             if self.pylab_gui_select is None:
@@ -3511,16 +3521,16 b' class InteractiveShell(SingletonConfigurable):'
                 print('Warning: Cannot change to a different GUI toolkit: %s.'
                         ' Using %s instead.' % (gui, self.pylab_gui_select))
                 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
-        
+
         pt.activate_matplotlib(backend)
         pt.configure_inline_support(self, backend)
-        
+
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         self.enable_gui(gui)
         self.magics_manager.registry['ExecutionMagics'].default_runner = \
             pt.mpl_runner(self.safe_execfile)
-        
+
         return gui, backend
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
@@ -3530,7 +3540,7 b' class InteractiveShell(SingletonConfigurable):'
         namespace all of numpy and pylab, and configures IPython to correctly
         interact with the GUI event loop.  The GUI backend to be used can be
         optionally selected with the optional ``gui`` argument.
-        
+
         This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
 
         Parameters
@@ -3550,9 +3560,9 b' class InteractiveShell(SingletonConfigurable):'
           This argument is ignored, no welcome message will be displayed.
         """
         from IPython.core.pylabtools import import_pylab
-        
+
         gui, backend = self.enable_matplotlib(gui)
-        
+
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and