Merge pull request #12734 from skalaydzhiyski/feature-add-autoreload-option-3

Feature add autoreload option 3

.git-blame-ignore-revs

@@ -13,3 +13,9 b''
 # <full commit hash>  # initial black-format 
 # <full commit hash>  # rename something internal
 6e748726282d1acb9a4f9f264ee679c474c4b8f5  # Apply pygrade --36plus on IPython/core/tests/test_inputtransformer.py.
+0233e65d8086d0ec34acb8685b7a5411633f0899 # apply pyupgrade to IPython/extensions/tests/test_autoreload.py
+a6a7e4dd7e51b892147895006d3a2a6c34b79ae6 # apply black to IPython/extensions/tests/test_autoreload.py
+c5ca5a8f25432dfd6b9eccbbe446a8348bf37cfa # apply pyupgrade to IPython/extensions/autoreload.py
+50624b84ccdece781750f5eb635a9efbf2fe30d6 # apply black to IPython/extensions/autoreload.py
+b7aaa47412b96379198705955004930c57f9d74a # apply pyupgrade to IPython/extensions/autoreload.py
+9c7476a88af3e567426b412f1b3c778401d8f6aa # apply black to IPython/extensions/autoreload.py

IPython/extensions/autoreload.py

@@ -48,6 +48,11 b' The following magic commands are provided:'
     Reload all modules (except those excluded by ``%aimport``) every
     time before executing the Python code typed.
 
+``%autoreload 3``
+
+    Reload all modules AND autoload newly added objects
+    every time before executing the Python code typed.
+
 ``%aimport``
 
     List modules which are to be automatically imported or not to be imported.
@@ -124,14 +129,18 b' from imp import reload'
 # Autoreload functionality
 #------------------------------------------------------------------------------
 
-class ModuleReloader(object):
+
+class ModuleReloader:
     enabled = False
     """Whether this reloader is enabled"""
 
     check_all = True
     """Autoreload all modules, not just those listed in 'modules'"""
 
-    def __init__(self):
+    autoload_obj = False
+    """Autoreload all modules AND autoload all new objects"""
+
+    def __init__(self, shell=None):
         # Modules that failed to reload: {module: mtime-on-failed-reload, ...}
         self.failed = {}
         # Modules specially marked as autoreloadable.
@@ -142,6 +151,7 b' class ModuleReloader(object):'
         self.old_objects = {}
         # Module modification timestamps
         self.modules_mtimes = {}
+        self.shell = shell
 
         # Cache module modification times
         self.check(check_all=True, do_reload=False)
@@ -176,22 +186,22 b' class ModuleReloader(object):'
         self.mark_module_reloadable(module_name)
 
         import_module(module_name)
-        top_name = module_name.split('.')[0]
+        top_name = module_name.split(".")[0]
         top_module = sys.modules[top_name]
         return top_module, top_name
 
     def filename_and_mtime(self, module):
-        if not hasattr(module, '__file__') or module.__file__ is None:
+        if not hasattr(module, "__file__") or module.__file__ is None:
             return None, None
 
-        if getattr(module, '__name__', None) in [None, '__mp_main__', '__main__']:
+        if getattr(module, "__name__", None) in [None, "__mp_main__", "__main__"]:
             # we cannot reload(__main__) or reload(__mp_main__)
             return None, None
 
         filename = module.__file__
         path, ext = os.path.splitext(filename)
 
-        if ext.lower() == '.py':
+        if ext.lower() == ".py":
             py_filename = filename
         else:
             try:
@@ -242,21 +252,35 b' class ModuleReloader(object):'
             # If we've reached this point, we should try to reload the module
             if do_reload:
                 try:
+                    if self.autoload_obj:
+                        superreload(m, reload, self.old_objects, self.shell)
+                    else:
                     superreload(m, reload, self.old_objects)
                     if py_filename in self.failed:
                         del self.failed[py_filename]
                 except:
-                    print("[autoreload of %s failed: %s]" % (
-                            modname, traceback.format_exc(10)), file=sys.stderr)
+                    print(
+                        "[autoreload of {} failed: {}]".format(
+                            modname, traceback.format_exc(10)
+                        ),
+                        file=sys.stderr,
+                    )
                     self.failed[py_filename] = pymtime
 
+
 #------------------------------------------------------------------------------
 # superreload
 #------------------------------------------------------------------------------
 
 
-func_attrs = ['__code__', '__defaults__', '__doc__',
-              '__closure__', '__globals__', '__dict__']
+func_attrs = [
+    "__code__",
+    "__defaults__",
+    "__doc__",
+    "__closure__",
+    "__globals__",
+    "__dict__",
+]
 
 
 def update_function(old, new):
@@ -299,7 +323,8 b' def update_class(old, new):'
                 pass
             continue
 
-        if update_generic(old_obj, new_obj): continue
+        if update_generic(old_obj, new_obj):
+            continue
 
         try:
             setattr(old, key, getattr(new, key))
@@ -329,16 +354,18 b' def isinstance2(a, b, typ):'
 
 
 UPDATE_RULES = [
-    (lambda a, b: isinstance2(a, b, type),
-     update_class),
-    (lambda a, b: isinstance2(a, b, types.FunctionType),
-     update_function),
-    (lambda a, b: isinstance2(a, b, property),
-     update_property),
+    (lambda a, b: isinstance2(a, b, type), update_class),
+    (lambda a, b: isinstance2(a, b, types.FunctionType), update_function),
+    (lambda a, b: isinstance2(a, b, property), update_property),
 ]
-UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
-                      lambda a, b: update_function(a.__func__, b.__func__)),
-])
+UPDATE_RULES.extend(
+    [
+        (
+            lambda a, b: isinstance2(a, b, types.MethodType),
+            lambda a, b: update_function(a.__func__, b.__func__),
+        ),
+    ]
+)
 
 
 def update_generic(a, b):
@@ -349,14 +376,45 b' def update_generic(a, b):'
     return False
 
 
-class StrongRef(object):
+class StrongRef:
     def __init__(self, obj):
         self.obj = obj
+
     def __call__(self):
         return self.obj
 
 
-def superreload(module, reload=reload, old_objects=None):
+mod_attrs = [
+    "__name__",
+    "__doc__",
+    "__package__",
+    "__loader__",
+    "__spec__",
+    "__file__",
+    "__cached__",
+    "__builtins__",
+]
+
+
+def append_obj(module, d, name, obj, autoload=False):
+    in_module = hasattr(obj, "__module__") and obj.__module__ == module.__name__
+    if autoload:
+        # check needed for module global built-ins
+        if not in_module and name in mod_attrs:
+            return False
+    else:
+        if not in_module:
+            return False
+
+    key = (module.__name__, name)
+    try:
+        d.setdefault(key, []).append(weakref.ref(obj))
+    except TypeError:
+        pass
+    return True
+
+
+def superreload(module, reload=reload, old_objects=None, shell=None):
     """Enhanced version of the builtin reload function.
 
     superreload remembers objects previously in the module, and
@@ -371,7 +429,7 b' def superreload(module, reload=reload, old_objects=None):'
 
     # collect old objects in the module
     for name, obj in list(module.__dict__.items()):
-        if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
+        if not append_obj(module, old_objects, name, obj):
             continue
         key = (module.__name__, name)
         try:
@@ -385,8 +443,8 b' def superreload(module, reload=reload, old_objects=None):'
         old_dict = module.__dict__.copy()
         old_name = module.__name__
         module.__dict__.clear()
-        module.__dict__['__name__'] = old_name
-        module.__dict__['__loader__'] = old_dict['__loader__']
+        module.__dict__["__name__"] = old_name
+        module.__dict__["__loader__"] = old_dict["__loader__"]
     except (TypeError, AttributeError, KeyError):
         pass
 
@@ -400,12 +458,21 b' def superreload(module, reload=reload, old_objects=None):'
     # iterate over all objects and update functions & classes
     for name, new_obj in list(module.__dict__.items()):
         key = (module.__name__, name)
-        if key not in old_objects: continue
+        if key not in old_objects:
+            # here 'shell' acts both as a flag and as an output var
+            if (
+                shell is None
+                or name == "Enum"
+                or not append_obj(module, old_objects, name, new_obj, True)
+            ):
+                continue
+            shell.user_ns[name] = new_obj
 
         new_refs = []
         for old_ref in old_objects[key]:
             old_obj = old_ref()
-            if old_obj is None: continue
+            if old_obj is None:
+                continue
             new_refs.append(old_ref)
             update_generic(old_obj, new_obj)
 
@@ -416,22 +483,25 b' def superreload(module, reload=reload, old_objects=None):'
 
     return module
 
+
 #------------------------------------------------------------------------------
 # IPython connectivity
 #------------------------------------------------------------------------------
 
 from IPython.core.magic import Magics, magics_class, line_magic
 
+
 @magics_class
 class AutoreloadMagics(Magics):
     def __init__(self, *a, **kw):
-        super(AutoreloadMagics, self).__init__(*a, **kw)
-        self._reloader = ModuleReloader()
+        super().__init__(*a, **kw)
+        self._reloader = ModuleReloader(self.shell)
         self._reloader.check_all = False
+        self._reloader.autoload_obj = False
         self.loaded_modules = set(sys.modules)
 
     @line_magic
-    def autoreload(self, parameter_s=''):
+    def autoreload(self, parameter_s=""):
         r"""%autoreload => Reload modules automatically
 
         %autoreload
@@ -475,19 +545,24 b' class AutoreloadMagics(Magics):'
           autoreloaded.
 
         """
-        if parameter_s == '':
+        if parameter_s == "":
             self._reloader.check(True)
-        elif parameter_s == '0':
+        elif parameter_s == "0":
             self._reloader.enabled = False
-        elif parameter_s == '1':
+        elif parameter_s == "1":
             self._reloader.check_all = False
             self._reloader.enabled = True
-        elif parameter_s == '2':
+        elif parameter_s == "2":
             self._reloader.check_all = True
             self._reloader.enabled = True
+            self._reloader.enabled = True
+        elif parameter_s == "3":
+            self._reloader.check_all = True
+            self._reloader.enabled = True
+            self._reloader.autoload_obj = True
 
     @line_magic
-    def aimport(self, parameter_s='', stream=None):
+    def aimport(self, parameter_s="", stream=None):
         """%aimport => Import modules for automatic reloading.
 
         %aimport
@@ -511,13 +586,13 b' class AutoreloadMagics(Magics):'
             if self._reloader.check_all:
                 stream.write("Modules to reload:\nall-except-skipped\n")
             else:
-                stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
-            stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
-        elif modname.startswith('-'):
+                stream.write("Modules to reload:\n%s\n" % " ".join(to_reload))
+            stream.write("\nModules to skip:\n%s\n" % " ".join(to_skip))
+        elif modname.startswith("-"):
             modname = modname[1:]
             self._reloader.mark_module_skipped(modname)
         else:
-            for _module in ([_.strip() for _ in modname.split(',')]):
+            for _module in [_.strip() for _ in modname.split(",")]:
                 top_module, top_name = self._reloader.aimport_module(_module)
 
                 # Inject module to user namespace
@@ -531,8 +606,7 b' class AutoreloadMagics(Magics):'
                 pass
 
     def post_execute_hook(self):
-        """Cache the modification times of any modules imported in this execution
-        """
+        """Cache the modification times of any modules imported in this execution"""
         newly_loaded_modules = set(sys.modules) - self.loaded_modules
         for modname in newly_loaded_modules:
             _, pymtime = self._reloader.filename_and_mtime(sys.modules[modname])
@@ -546,5 +620,5 b' def load_ipython_extension(ip):'
     """Load the extension in IPython."""
     auto_reload = AutoreloadMagics(ip)
     ip.register_magics(auto_reload)
-    ip.events.register('pre_run_cell', auto_reload.pre_run_cell)
-    ip.events.register('post_execute', auto_reload.post_execute_hook)
+    ip.events.register("pre_run_cell", auto_reload.pre_run_cell)
+    ip.events.register("post_execute", auto_reload.post_execute_hook)

IPython/extensions/tests/test_autoreload.py

@@ -35,20 +35,20 b' from IPython.core.events import EventManager, pre_run_cell'
 
 noop = lambda *a, **kw: None
 
-class FakeShell:
 
+class FakeShell:
     def __init__(self):
         self.ns = {}
         self.user_ns = self.ns
         self.user_ns_hidden = {}
-        self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
+        self.events = EventManager(self, {"pre_run_cell", pre_run_cell})
         self.auto_magics = AutoreloadMagics(shell=self)
-        self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
+        self.events.register("pre_run_cell", self.auto_magics.pre_run_cell)
 
     register_magics = set_hook = noop
 
     def run_code(self, code):
-        self.events.trigger('pre_run_cell')
+        self.events.trigger("pre_run_cell")
         exec(code, self.user_ns)
         self.auto_magics.post_execute_hook()
 
@@ -111,20 +111,22 b' class Fixture(TestCase):'
         time.sleep(1.05)
 
         # Write
-        with open(filename, 'w') as f:
+        with open(filename, "w") as f:
             f.write(content)
 
     def new_module(self, code):
         code = textwrap.dedent(code)
         mod_name, mod_fn = self.get_module()
-        with open(mod_fn, 'w') as f:
+        with open(mod_fn, "w") as f:
             f.write(code)
         return mod_name, mod_fn
 
+
 #-----------------------------------------------------------------------------
 # Test automatic reloading
 #-----------------------------------------------------------------------------
 
+
 def pickle_get_current_class(obj):
     """
     Original issue comes from pickle; hence the name.
@@ -136,25 +138,36 b' def pickle_get_current_class(obj):'
         obj2 = getattr(obj2, subpath)
     return obj2
 
-class TestAutoreload(Fixture):
 
+class TestAutoreload(Fixture):
     def test_reload_enums(self):
-        mod_name, mod_fn = self.new_module(textwrap.dedent("""
+        mod_name, mod_fn = self.new_module(
+            textwrap.dedent(
+                """
                                 from enum import Enum
                                 class MyEnum(Enum):
                                     A = 'A'
                                     B = 'B'
-                            """))
+                            """
+            )
+        )
         self.shell.magic_autoreload("2")
         self.shell.magic_aimport(mod_name)
-        self.write_file(mod_fn, textwrap.dedent("""
+        self.write_file(
+            mod_fn,
+            textwrap.dedent(
+                """
                                 from enum import Enum
                                 class MyEnum(Enum):
                                     A = 'A'
                                     B = 'B'
                                     C = 'C'
-                            """))
-        with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
+                            """
+            ),
+        )
+        with tt.AssertNotPrints(
+            ("[autoreload of %s failed:" % mod_name), channel="stderr"
+        ):
             self.shell.run_code("pass")  # trigger another reload
 
     def test_reload_class_type(self):
@@ -195,7 +208,9 b' class TestAutoreload(Fixture):'
 
     def test_reload_class_attributes(self):
         self.shell.magic_autoreload("2")
-        mod_name, mod_fn = self.new_module(textwrap.dedent("""
+        mod_name, mod_fn = self.new_module(
+            textwrap.dedent(
+                """
                                 class MyClass:
 
                                     def __init__(self, a=10):
@@ -241,16 +256,99 b' class TestAutoreload(Fixture):'
 
         self.shell.run_code("second = MyClass(5)")
 
-        for object_name in {'first', 'second'}:
-            self.shell.run_code("{object_name}.power(5)".format(object_name=object_name))
+        for object_name in {"first", "second"}:
+            self.shell.run_code(f"{object_name}.power(5)")
             with nt.assert_raises(AttributeError):
-                self.shell.run_code("{object_name}.cube()".format(object_name=object_name))
+                self.shell.run_code(f"{object_name}.cube()")
             with nt.assert_raises(AttributeError):
-                self.shell.run_code("{object_name}.square()".format(object_name=object_name))
-            self.shell.run_code("{object_name}.b".format(object_name=object_name))
-            self.shell.run_code("{object_name}.a".format(object_name=object_name))
+                self.shell.run_code(f"{object_name}.square()")
+            self.shell.run_code(f"{object_name}.b")
+            self.shell.run_code(f"{object_name}.a")
             with nt.assert_raises(AttributeError):
-                self.shell.run_code("{object_name}.toto".format(object_name=object_name))
+                self.shell.run_code(f"{object_name}.toto")
+
+    def test_autoload_newly_added_objects(self):
+        self.shell.magic_autoreload("3")
+        mod_code = """
+        def func1(): pass
+        """
+        mod_name, mod_fn = self.new_module(textwrap.dedent(mod_code))
+        self.shell.run_code(f"from {mod_name} import *")
+        self.shell.run_code("func1()")
+        with nt.assert_raises(NameError):
+            self.shell.run_code("func2()")
+        with nt.assert_raises(NameError):
+            self.shell.run_code("t = Test()")
+        with nt.assert_raises(NameError):
+            self.shell.run_code("number")
+
+        # ----------- TEST NEW OBJ LOADED --------------------------
+
+        new_code = """
+        def func1(): pass
+        def func2(): pass
+        class Test: pass
+        number = 0
+        from enum import Enum
+        class TestEnum(Enum):
+            A = 'a'
+        """
+        self.write_file(mod_fn, textwrap.dedent(new_code))
+
+        # test function now exists in shell's namespace namespace
+        self.shell.run_code("func2()")
+        # test function now exists in module's dict
+        self.shell.run_code(f"import sys; sys.modules['{mod_name}'].func2()")
+        # test class now exists
+        self.shell.run_code("t = Test()")
+        # test global built-in var now exists
+        self.shell.run_code("number")
+        # test the enumerations gets loaded succesfully
+        self.shell.run_code("TestEnum.A")
+
+        # ----------- TEST NEW OBJ CAN BE CHANGED --------------------
+
+        new_code = """
+        def func1(): return 'changed'
+        def func2(): return 'changed'
+        class Test:
+            def new_func(self):
+                return 'changed'
+        number = 1
+        from enum import Enum
+        class TestEnum(Enum):
+            A = 'a'
+            B = 'added'
+        """
+        self.write_file(mod_fn, textwrap.dedent(new_code))
+        self.shell.run_code("assert func1() == 'changed'")
+        self.shell.run_code("assert func2() == 'changed'")
+        self.shell.run_code("t = Test(); assert t.new_func() == 'changed'")
+        self.shell.run_code("assert number == 1")
+        self.shell.run_code("assert TestEnum.B.value == 'added'")
+
+        # ----------- TEST IMPORT FROM MODULE --------------------------
+
+        new_mod_code = """
+        from enum import Enum
+        class Ext(Enum):
+            A = 'ext'
+        def ext_func():
+            return 'ext'
+        class ExtTest:
+            def meth(self):
+                return 'ext'
+        ext_int = 2
+        """
+        new_mod_name, new_mod_fn = self.new_module(textwrap.dedent(new_mod_code))
+        current_mod_code = f"""
+        from {new_mod_name} import *
+        """
+        self.write_file(mod_fn, textwrap.dedent(current_mod_code))
+        self.shell.run_code("assert Ext.A.value == 'ext'")
+        self.shell.run_code("assert ext_func() == 'ext'")
+        self.shell.run_code("t = ExtTest(); assert t.meth() == 'ext'")
+        self.shell.run_code("assert ext_int == 2")
 
     def _check_smoketest(self, use_aimport=True):
         """
@@ -258,7 +356,8 b' class TestAutoreload(Fixture):'
         '%autoreload 1' or '%autoreload 2'
         """
 
-        mod_name, mod_fn = self.new_module("""
+        mod_name, mod_fn = self.new_module(
+            """
 x = 9
 
 z = 123  # this item will be deleted
@@ -281,7 +380,8 b' class Baz(object):'
 class Bar:    # old-style class: weakref doesn't work for it on Python < 2.7
     def foo(self):
         return 1
-""")
+"""
+        )
 
         #
         # Import module, and mark for reloading
@@ -300,8 +400,9 b" class Bar: # old-style class: weakref doesn't work for it on Python < 2.7"
             self.shell.run_code("import %s" % mod_name)
             stream = StringIO()
             self.shell.magic_aimport("", stream=stream)
-            nt.assert_true("Modules to reload:\nall-except-skipped" in
-                           stream.getvalue())
+            nt.assert_true(
+                "Modules to reload:\nall-except-skipped" in stream.getvalue()
+            )
         nt.assert_in(mod_name, self.shell.ns)
 
         mod = sys.modules[mod_name]
@@ -336,20 +437,29 b" class Bar: # old-style class: weakref doesn't work for it on Python < 2.7"
         # Simulate a failed reload: no reload should occur and exactly
         # one error message should be printed
         #
-        self.write_file(mod_fn, """
+        self.write_file(
+            mod_fn,
+            """
 a syntax error
-""")
+""",
+        )
 
-        with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
+        with tt.AssertPrints(
+            ("[autoreload of %s failed:" % mod_name), channel="stderr"
+        ):
             self.shell.run_code("pass") # trigger reload
-        with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
+        with tt.AssertNotPrints(
+            ("[autoreload of %s failed:" % mod_name), channel="stderr"
+        ):
             self.shell.run_code("pass") # trigger another reload
         check_module_contents()
 
         #
         # Rewrite module (this time reload should succeed)
         #
-        self.write_file(mod_fn, """
+        self.write_file(
+            mod_fn,
+            """
 x = 10
 
 def foo(y):
@@ -367,11 +477,12 b' class Baz(object):'
 class Bar:    # old-style class
     def foo(self):
         return 2
-""")
+""",
+        )
 
         def check_module_contents():
             nt.assert_equal(mod.x, 10)
-            nt.assert_false(hasattr(mod, 'z'))
+            nt.assert_false(hasattr(mod, "z"))
 
             nt.assert_equal(old_foo(0), 4) # superreload magic!
             nt.assert_equal(mod.foo(0), 4)
@@ -383,8 +494,8 b' class Bar: # old-style class'
             nt.assert_equal(old_obj.quux, 43)
             nt.assert_equal(obj.quux, 43)
 
-            nt.assert_false(hasattr(old_obj, 'zzz'))
-            nt.assert_false(hasattr(obj, 'zzz'))
+            nt.assert_false(hasattr(old_obj, "zzz"))
+            nt.assert_false(hasattr(obj, "zzz"))
 
             obj2 = mod.Bar()
             nt.assert_equal(old_obj2.foo(), 2)
@@ -408,17 +519,19 b' class Bar: # old-style class'
             self.shell.magic_aimport("-" + mod_name)
             stream = StringIO()
             self.shell.magic_aimport("", stream=stream)
-            nt.assert_true(("Modules to skip:\n%s" % mod_name) in
-                           stream.getvalue())
+            nt.assert_true(("Modules to skip:\n%s" % mod_name) in stream.getvalue())
 
             # This should succeed, although no such module exists
             self.shell.magic_aimport("-tmpmod_as318989e89ds")
         else:
             self.shell.magic_autoreload("0")
 
-        self.write_file(mod_fn, """
+        self.write_file(
+            mod_fn,
+            """
 x = -99
-""")
+""",
+        )
 
         self.shell.run_code("pass") # trigger reload
         self.shell.run_code("pass")
@@ -440,8 +553,3 b' x = -99'
 
     def test_smoketest_autoreload(self):
         self._check_smoketest(use_aimport=False)
-
-
-
-
-

docs/source/whatsnew/development.rst

@@ -152,6 +152,24 b' and "??", in much the same way it can be done when using the IPython prompt::'
 
 Previously, "pinfo" or "pinfo2" command had to be used for this purpose.
 
+
+Autoreload 3 feature
+====================
+
+Example: When an IPython session is ran with the 'autoreload' extension loaded,
+you will now have the option '3' to select which means the following:
+
+    1. replicate all functionality from option 2
+    2. autoload all new funcs/classes/enums/globals from the module when they're added
+    3. autoload all newly imported funcs/classes/enums/globals from external modules
+
+Try ``%autoreload 3`` in an IPython session after running ``%load_ext autoreload``
+
+For more information please see unit test -
+    extensions/tests/test_autoreload.py : 'test_autoload_newly_added_objects'
+
+=======
+
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT.
 
 As a reminder, IPython master has diverged from the 7.x branch, thus master may