diff --git a/mercurial/phases.py b/mercurial/phases.py
--- a/mercurial/phases.py
+++ b/mercurial/phases.py
@@ -645,11 +645,6 @@ class phasecache:
         return changes
 
     def retractboundary(self, repo, tr, targetphase, nodes):
-        oldroots = {
-            phase: revs
-            for phase, revs in self._phaseroots.items()
-            if phase <= targetphase
-        }
         if tr is None:
             phasetracking = None
         else:
@@ -657,22 +652,8 @@ class phasecache:
         repo = repo.unfiltered()
         retracted = self._retractboundary(repo, tr, targetphase, nodes)
         if retracted and phasetracking is not None:
-
-            # find the affected revisions
-            new = self._phaseroots[targetphase]
-            old = oldroots[targetphase]
-            affected = set(repo.revs(b'(%ld::) - (%ld::)', new, old))
-
-            # find the phase of the affected revision
-            for phase in range(targetphase, -1, -1):
-                if phase:
-                    roots = oldroots.get(phase, [])
-                    revs = set(repo.revs(b'%ld::%ld', roots, affected))
-                    affected -= revs
-                else:  # public phase
-                    revs = affected
-                for r in sorted(revs):
-                    _trackphasechange(phasetracking, r, phase, targetphase)
+            for r, old_phase in sorted(retracted.items()):
+                _trackphasechange(phasetracking, r, old_phase, targetphase)
         repo.invalidatevolatilesets()
 
     def _retractboundary(self, repo, tr, targetphase, nodes=None, revs=None):
diff --git a/tests/test-phases-exchange.t b/tests/test-phases-exchange.t
--- a/tests/test-phases-exchange.t
+++ b/tests/test-phases-exchange.t
@@ -1595,7 +1595,6 @@ abort behavior
 trying to push a secret changeset doesn't confuse auto-publish
 
   $ hg phase --secret --force
-  test-debug-phase: move rev 0: 1 -> 2
   test-debug-phase: move rev 1: 1 -> 2
 
   $ hg push --config experimental.auto-publish=abort
diff --git a/tests/test-phases.t b/tests/test-phases.t
--- a/tests/test-phases.t
+++ b/tests/test-phases.t
@@ -54,9 +54,7 @@ Following commit are draft too
 Working directory phase is secret when its parent is secret.
 
   $ hg phase --force --secret .
-  test-debug-phase: move rev 0: 1 -> 2
   test-debug-phase: move rev 1: 1 -> 2
-  test-hook-close-phase: 4a2df7238c3b48766b5e22fafbb8a2f506ec8256:  draft -> secret
   test-hook-close-phase: 27547f69f25460a52fff66ad004e58da7ad3fb56:  draft -> secret
   $ hg log -r 'wdir()' -T '{phase}\n'
   secret