From 0725b4e76fb1b1980c1cc30f4ad0022c588316ac 2023-02-17 17:01:11 From: Fernando Perez Date: 2023-02-17 17:01:11 Subject: [PATCH] Shaperilio/autoreload verbosity (#13774) Worked on three things: 1. More descriptive parameter names for `%autoreload`; `now`, `off`, `explicit`, `all`, `complete`. (This last one could probably use a better name, but I couldn't think of anything better based on the message in 1d3018a93e98ad55f41d4419f835b738de80e1b7) 2. New optional arguments for `%autoreload` allow displaying the names of modules that are reloaded. Use `--print` or `-p` to use `print` statements, or `--log` / `-l` to log at `INFO` level. 3. `%aimport` can parse whitelist/blacklist modules on the same line, e.g. `%aimport os, -math` now works. `%autoreload` and will also now raise a `ValueError` if the parameter is invalid. I suppose a bit more verification could be done for input to `%aimport`.... --- diff --git a/IPython/extensions/autoreload.py b/IPython/extensions/autoreload.py index f485ac3..cb63a55 100644 --- a/IPython/extensions/autoreload.py +++ b/IPython/extensions/autoreload.py @@ -29,29 +29,33 @@ Usage The following magic commands are provided: -``%autoreload`` +``%autoreload``, ``%autoreload now`` Reload all modules (except those excluded by ``%aimport``) automatically now. -``%autoreload 0`` +``%autoreload 0``, ``%autoreload off`` Disable automatic reloading. -``%autoreload 1`` +``%autoreload 1``, ``%autoreload explicit`` Reload all modules imported with ``%aimport`` every time before executing the Python code typed. -``%autoreload 2`` +``%autoreload 2``, ``%autoreload all`` Reload all modules (except those excluded by ``%aimport``) every time before executing the Python code typed. -``%autoreload 3`` +``%autoreload 3``, ``%autoreload complete`` - Reload all modules AND autoload newly added objects - every time before executing the Python code typed. + Same as 2/all, but also adds any new objects in the module. See + unit test at IPython/extensions/tests/test_autoreload.py::test_autoload_newly_added_objects + + Adding ``--print`` or ``-p`` to the ``%autoreload`` line will print autoreload activity to + standard out. ``--log`` or ``-l`` will do it to the log at INFO level; both can be used + simultaneously. ``%aimport`` @@ -101,6 +105,9 @@ Some of the known remaining caveats are: - Reloading a module, or importing the same module by a different name, creates new Enums. These may look the same, but are not. """ +from IPython.core import magic_arguments +from IPython.core.magic import Magics, magics_class, line_magic + __skip_doctest__ = True # ----------------------------------------------------------------------------- @@ -125,6 +132,7 @@ import traceback import types import weakref import gc +import logging from importlib import import_module, reload from importlib.util import source_from_cache @@ -156,6 +164,9 @@ class ModuleReloader: self.modules_mtimes = {} self.shell = shell + # Reporting callable for verbosity + self._report = lambda msg: None # by default, be quiet. + # Cache module modification times self.check(check_all=True, do_reload=False) @@ -254,6 +265,7 @@ class ModuleReloader: # If we've reached this point, we should try to reload the module if do_reload: + self._report(f"Reloading '{modname}'.") try: if self.autoload_obj: superreload(m, reload, self.old_objects, self.shell) @@ -495,8 +507,6 @@ def superreload(module, reload=reload, old_objects=None, shell=None): # IPython connectivity # ------------------------------------------------------------------------------ -from IPython.core.magic import Magics, magics_class, line_magic - @magics_class class AutoreloadMagics(Magics): @@ -508,24 +518,67 @@ class AutoreloadMagics(Magics): self.loaded_modules = set(sys.modules) @line_magic - def autoreload(self, parameter_s=""): + @magic_arguments.magic_arguments() + @magic_arguments.argument( + "mode", + type=str, + default="now", + nargs="?", + help="""blank or 'now' - Reload all modules (except those excluded by %%aimport) + automatically now. + + '0' or 'off' - Disable automatic reloading. + + '1' or 'explicit' - Reload only modules imported with %%aimport every + time before executing the Python code typed. + + '2' or 'all' - Reload all modules (except those excluded by %%aimport) + every time before executing the Python code typed. + + '3' or 'complete' - Same as 2/all, but also but also adds any new + objects in the module. + """, + ) + @magic_arguments.argument( + "-p", + "--print", + action="store_true", + default=False, + help="Show autoreload activity using `print` statements", + ) + @magic_arguments.argument( + "-l", + "--log", + action="store_true", + default=False, + help="Show autoreload activity using the logger", + ) + def autoreload(self, line=""): r"""%autoreload => Reload modules automatically - %autoreload + %autoreload or %autoreload now Reload all modules (except those excluded by %aimport) automatically now. - %autoreload 0 + %autoreload 0 or %autoreload off Disable automatic reloading. - %autoreload 1 - Reload all modules imported with %aimport every time before executing + %autoreload 1 or %autoreload explicit + Reload only modules imported with %aimport every time before executing the Python code typed. - %autoreload 2 + %autoreload 2 or %autoreload all Reload all modules (except those excluded by %aimport) every time before executing the Python code typed. + %autoreload 3 or %autoreload complete + Same as 2/all, but also but also adds any new objects in the module. See + unit test at IPython/extensions/tests/test_autoreload.py::test_autoload_newly_added_objects + + The optional arguments --print and --log control display of autoreload activity. The default + is to act silently; --print (or -p) will print out the names of modules that are being + reloaded, and --log (or -l) outputs them to the log at INFO level. + Reloading Python modules in a reliable way is in general difficult, and unexpected things may occur. %autoreload tries to work around common pitfalls by replacing function code objects and @@ -552,21 +605,47 @@ class AutoreloadMagics(Magics): autoreloaded. """ - if parameter_s == "": + args = magic_arguments.parse_argstring(self.autoreload, line) + mode = args.mode.lower() + + p = print + + logger = logging.getLogger("autoreload") + + l = logger.info + + def pl(msg): + p(msg) + l(msg) + + if args.print is False and args.log is False: + self._reloader._report = lambda msg: None + elif args.print is True: + if args.log is True: + self._reloader._report = pl + else: + self._reloader._report = p + elif args.log is True: + self._reloader._report = l + + if mode == "" or mode == "now": self._reloader.check(True) - elif parameter_s == "0": + elif mode == "0" or mode == "off": self._reloader.enabled = False - elif parameter_s == "1": + elif mode == "1" or mode == "explicit": + self._reloader.enabled = True self._reloader.check_all = False + self._reloader.autoload_obj = False + elif mode == "2" or mode == "all": self._reloader.enabled = True - elif parameter_s == "2": self._reloader.check_all = True + self._reloader.autoload_obj = False + elif mode == "3" or mode == "complete": self._reloader.enabled = True - self._reloader.enabled = True - elif parameter_s == "3": self._reloader.check_all = True - self._reloader.enabled = True self._reloader.autoload_obj = True + else: + raise ValueError(f'Unrecognized autoreload mode "{mode}".') @line_magic def aimport(self, parameter_s="", stream=None): @@ -576,13 +655,14 @@ class AutoreloadMagics(Magics): List modules to automatically import and not to import. %aimport foo - Import module 'foo' and mark it to be autoreloaded for %autoreload 1 + Import module 'foo' and mark it to be autoreloaded for %autoreload explicit %aimport foo, bar - Import modules 'foo', 'bar' and mark them to be autoreloaded for %autoreload 1 + Import modules 'foo', 'bar' and mark them to be autoreloaded for %autoreload explicit - %aimport -foo - Mark module 'foo' to not be autoreloaded for %autoreload 1 + %aimport -foo, bar + Mark module 'foo' to not be autoreloaded for %autoreload explicit, all, or complete, and 'bar' + to be autoreloaded for mode explicit. """ modname = parameter_s if not modname: @@ -595,15 +675,16 @@ class AutoreloadMagics(Magics): else: stream.write("Modules to reload:\n%s\n" % " ".join(to_reload)) stream.write("\nModules to skip:\n%s\n" % " ".join(to_skip)) - elif modname.startswith("-"): - modname = modname[1:] - self._reloader.mark_module_skipped(modname) else: for _module in [_.strip() for _ in modname.split(",")]: - top_module, top_name = self._reloader.aimport_module(_module) - - # Inject module to user namespace - self.shell.push({top_name: top_module}) + if _module.startswith("-"): + _module = _module[1:].strip() + self._reloader.mark_module_skipped(_module) + else: + top_module, top_name = self._reloader.aimport_module(_module) + + # Inject module to user namespace + self.shell.push({top_name: top_module}) def pre_run_cell(self): if self._reloader.enabled: diff --git a/IPython/extensions/tests/test_autoreload.py b/IPython/extensions/tests/test_autoreload.py index 2c3c9db..89a4add 100644 --- a/IPython/extensions/tests/test_autoreload.py +++ b/IPython/extensions/tests/test_autoreload.py @@ -22,6 +22,7 @@ import shutil import random import time from io import StringIO +from dataclasses import dataclass import IPython.testing.tools as tt @@ -310,6 +311,7 @@ class TestAutoreload(Fixture): self.shell.run_code("pass") # trigger another reload def test_autoload_newly_added_objects(self): + # All of these fail with %autoreload 2 self.shell.magic_autoreload("3") mod_code = """ def func1(): pass @@ -393,6 +395,95 @@ class TestAutoreload(Fixture): self.shell.run_code("t = ExtTest(); assert t.meth() == 'ext'") self.shell.run_code("assert ext_int == 2") + def test_verbose_names(self): + # Asserts correspondense between original mode names and their verbose equivalents. + @dataclass + class AutoreloadSettings: + check_all: bool + enabled: bool + autoload_obj: bool + + def gather_settings(mode): + self.shell.magic_autoreload(mode) + module_reloader = self.shell.auto_magics._reloader + return AutoreloadSettings( + module_reloader.check_all, + module_reloader.enabled, + module_reloader.autoload_obj, + ) + + assert gather_settings("0") == gather_settings("off") + assert gather_settings("0") == gather_settings("OFF") # Case insensitive + assert gather_settings("1") == gather_settings("explicit") + assert gather_settings("2") == gather_settings("all") + assert gather_settings("3") == gather_settings("complete") + + # And an invalid mode name raises an exception. + with self.assertRaises(ValueError): + self.shell.magic_autoreload("4") + + def test_aimport_parsing(self): + # Modules can be included or excluded all in one line. + module_reloader = self.shell.auto_magics._reloader + self.shell.magic_aimport("os") # import and mark `os` for auto-reload. + assert module_reloader.modules["os"] is True + assert "os" not in module_reloader.skip_modules.keys() + + self.shell.magic_aimport("-math") # forbid autoreloading of `math` + assert module_reloader.skip_modules["math"] is True + assert "math" not in module_reloader.modules.keys() + + self.shell.magic_aimport( + "-os, math" + ) # Can do this all in one line; wasn't possible before. + assert module_reloader.modules["math"] is True + assert "math" not in module_reloader.skip_modules.keys() + assert module_reloader.skip_modules["os"] is True + assert "os" not in module_reloader.modules.keys() + + def test_autoreload_output(self): + self.shell.magic_autoreload("complete") + mod_code = """ + def func1(): pass + """ + mod_name, mod_fn = self.new_module(mod_code) + self.shell.run_code(f"import {mod_name}") + with tt.AssertPrints("", channel="stdout"): # no output; this is default + self.shell.run_code("pass") + + self.shell.magic_autoreload("complete --print") + self.write_file(mod_fn, mod_code) # "modify" the module + with tt.AssertPrints( + f"Reloading '{mod_name}'.", channel="stdout" + ): # see something printed out + self.shell.run_code("pass") + + self.shell.magic_autoreload("complete -p") + self.write_file(mod_fn, mod_code) # "modify" the module + with tt.AssertPrints( + f"Reloading '{mod_name}'.", channel="stdout" + ): # see something printed out + self.shell.run_code("pass") + + self.shell.magic_autoreload("complete --print --log") + self.write_file(mod_fn, mod_code) # "modify" the module + with tt.AssertPrints( + f"Reloading '{mod_name}'.", channel="stdout" + ): # see something printed out + self.shell.run_code("pass") + + self.shell.magic_autoreload("complete --print --log") + self.write_file(mod_fn, mod_code) # "modify" the module + with self.assertLogs(logger="autoreload") as lo: # see something printed out + self.shell.run_code("pass") + assert lo.output == [f"INFO:autoreload:Reloading '{mod_name}'."] + + self.shell.magic_autoreload("complete -l") + self.write_file(mod_fn, mod_code) # "modify" the module + with self.assertLogs(logger="autoreload") as lo: # see something printed out + self.shell.run_code("pass") + assert lo.output == [f"INFO:autoreload:Reloading '{mod_name}'."] + def _check_smoketest(self, use_aimport=True): """ Functional test for the automatic reloader using either diff --git a/docs/source/whatsnew/pr/autoreload-verbosity.rst b/docs/source/whatsnew/pr/autoreload-verbosity.rst new file mode 100644 index 0000000..68ff300 --- /dev/null +++ b/docs/source/whatsnew/pr/autoreload-verbosity.rst @@ -0,0 +1,26 @@ +Autoreload verbosity +==================== + +We introduce more descriptive names for the ``%autoreload`` parameter: + +- ``%autoreload now`` (also ``%autoreload``) - perform autoreload immediately. +- ``%autoreload off`` (also ``%autoreload 0``) - turn off autoreload. +- ``%autoreload explicit`` (also ``%autoreload 1``) - turn on autoreload only for modules + whitelisted by ``%aimport`` statements. +- ``%autoreload all`` (also ``%autoreload 2``) - turn on autoreload for all modules except those + blacklisted by ``%aimport`` statements. +- ``%autoreload complete`` (also ``%autoreload 3``) - all the fatures of ``all`` but also adding new + objects from the imported modules (see + IPython/extensions/tests/test_autoreload.py::test_autoload_newly_added_objects). + +The original designations (e.g. "2") still work, and these new ones are case-insensitive. + +Additionally, the option ``--print`` or ``-p`` can be added to the line to print the names of +modules being reloaded. Similarly, ``--log`` or ``-l`` will output the names to the logger at INFO +level. Both can be used simultaneously. + +The parsing logic for ``%aimport`` is now improved such that modules can be whitelisted and +blacklisted in the same line, e.g. it's now possible to call ``%aimport os, -math`` to include +``os`` for ``%autoreload explicit`` and exclude ``math`` for modes ``all`` and ``complete``. + +