# HG changeset patch # User Simon Sapin # Date 2021-05-19 16:35:43 # Node ID a43d256c041a59b9dc5859d79e3f28aebabd9180 # Parent e985a36c2aa383f98f9a303013efe6ad0bd24ff0 dirstate-v2: Add `hg debugupgraderepo` command support This command changes changes the file formats used inside an existing repository to what they would be in a new repository with the current config. For example: hg debugupgraderepo --config format.exp-dirstate-v2=1 --run hg debugupgraderepo --config format.exp-dirstate-v2=0 --run If a repository has a dirstate in v1 format, the first command would upgrade it to dirstate-v2. Conversely, if a repository has a dirstate in v2 format, the second command would downgrade it to v1. (Both may also run some unrelated upgrades.) Since `format.exp-dirstate-v2` is currently disabled by default, not specifying it in `--config` or any configuration file would result in the second command. Differential Revision: https://phab.mercurial-scm.org/D10769 diff --git a/mercurial/dirstate.py b/mercurial/dirstate.py --- a/mercurial/dirstate.py +++ b/mercurial/dirstate.py @@ -1775,6 +1775,12 @@ if rustmod is not None: # for consistent view between _pl() and _read() invocations self._pendingmode = None + self._use_dirstate_tree = self._ui.configbool( + b"experimental", + b"dirstate-tree.in-memory", + False, + ) + def addfile(self, *args, **kwargs): return self._rustmap.addfile(*args, **kwargs) @@ -1903,13 +1909,8 @@ if rustmod is not None: raise st = b'' - use_dirstate_tree = self._ui.configbool( - b"experimental", - b"dirstate-tree.in-memory", - False, - ) self._rustmap, parents = rustmod.DirstateMap.new( - use_dirstate_tree, self._use_dirstate_v2, st + self._use_dirstate_tree, self._use_dirstate_v2, st ) if parents and not self._dirtyparents: diff --git a/mercurial/upgrade_utils/actions.py b/mercurial/upgrade_utils/actions.py --- a/mercurial/upgrade_utils/actions.py +++ b/mercurial/upgrade_utils/actions.py @@ -80,7 +80,7 @@ class improvement(object): # operation in which this improvement was removed postdowngrademessage = None - # By default for now, we assume every improvement touches all the things + # By default we assume that every improvement touches requirements and all revlogs # Whether this improvement touches filelogs touches_filelogs = True @@ -94,6 +94,9 @@ class improvement(object): # Whether this improvement changes repository requirements touches_requirements = True + # Whether this improvement touches the dirstate + touches_dirstate = False + allformatvariant = [] # type: List[Type['formatvariant']] @@ -167,6 +170,27 @@ class fncache(requirementformatvariant): @registerformatvariant +class dirstatev2(requirementformatvariant): + name = b'dirstate-v2' + _requirement = requirements.DIRSTATE_V2_REQUIREMENT + + default = False + + description = _( + b'version 1 of the dirstate file format requires ' + b'reading and parsing it all at once.' + ) + + upgrademessage = _(b'"hg status" will be faster') + + touches_filelogs = False + touches_manifests = False + touches_changelog = False + touches_requirements = True + touches_dirstate = True + + +@registerformatvariant class dotencode(requirementformatvariant): name = b'dotencode' @@ -644,7 +668,6 @@ class UpgradeOperation(object): self.current_requirements = current_requirements # list of upgrade actions the operation will perform self.upgrade_actions = upgrade_actions - self._upgrade_actions_names = set([a.name for a in upgrade_actions]) self.removed_actions = removed_actions self.revlogs_to_process = revlogs_to_process # requirements which will be added by the operation @@ -667,41 +690,42 @@ class UpgradeOperation(object): ] # delta reuse mode of this upgrade operation + upgrade_actions_names = self.upgrade_actions_names self.delta_reuse_mode = revlog.revlog.DELTAREUSEALWAYS - if b're-delta-all' in self._upgrade_actions_names: + if b're-delta-all' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSENEVER - elif b're-delta-parent' in self._upgrade_actions_names: + elif b're-delta-parent' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSESAMEREVS - elif b're-delta-multibase' in self._upgrade_actions_names: + elif b're-delta-multibase' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSESAMEREVS - elif b're-delta-fulladd' in self._upgrade_actions_names: + elif b're-delta-fulladd' in upgrade_actions_names: self.delta_reuse_mode = revlog.revlog.DELTAREUSEFULLADD # should this operation force re-delta of both parents self.force_re_delta_both_parents = ( - b're-delta-multibase' in self._upgrade_actions_names + b're-delta-multibase' in upgrade_actions_names ) # should this operation create a backup of the store self.backup_store = backup_store - # whether the operation touches different revlogs at all or not - self.touches_filelogs = self._touches_filelogs() - self.touches_manifests = self._touches_manifests() - self.touches_changelog = self._touches_changelog() - # whether the operation touches requirements file or not - self.touches_requirements = self._touches_requirements() - self.touches_store = ( - self.touches_filelogs - or self.touches_manifests - or self.touches_changelog - ) + @property + def upgrade_actions_names(self): + return set([a.name for a in self.upgrade_actions]) + + @property + def requirements_only(self): # does the operation only touches repository requirement - self.requirements_only = ( - self.touches_requirements and not self.touches_store + return ( + self.touches_requirements + and not self.touches_filelogs + and not self.touches_manifests + and not self.touches_changelog + and not self.touches_dirstate ) - def _touches_filelogs(self): + @property + def touches_filelogs(self): for a in self.upgrade_actions: # in optimisations, we re-process the revlogs again if a.type == OPTIMISATION: @@ -713,7 +737,8 @@ class UpgradeOperation(object): return True return False - def _touches_manifests(self): + @property + def touches_manifests(self): for a in self.upgrade_actions: # in optimisations, we re-process the revlogs again if a.type == OPTIMISATION: @@ -725,7 +750,8 @@ class UpgradeOperation(object): return True return False - def _touches_changelog(self): + @property + def touches_changelog(self): for a in self.upgrade_actions: # in optimisations, we re-process the revlogs again if a.type == OPTIMISATION: @@ -737,7 +763,8 @@ class UpgradeOperation(object): return True return False - def _touches_requirements(self): + @property + def touches_requirements(self): for a in self.upgrade_actions: # optimisations are used to re-process revlogs and does not result # in a requirement being added or removed @@ -749,6 +776,18 @@ class UpgradeOperation(object): if a.touches_requirements: return True + @property + def touches_dirstate(self): + for a in self.upgrade_actions: + # revlog optimisations do not affect the dirstate + if a.type == OPTIMISATION: + pass + elif a.touches_dirstate: + return True + for a in self.removed_actions: + if a.touches_dirstate: + return True + return False def _write_labeled(self, l, label): @@ -908,6 +947,7 @@ def supportremovedrequirements(repo): requirements.REVLOGV2_REQUIREMENT, requirements.CHANGELOGV2_REQUIREMENT, requirements.REVLOGV1_REQUIREMENT, + requirements.DIRSTATE_V2_REQUIREMENT, } for name in compression.compengines: engine = compression.compengines[name] @@ -970,6 +1010,7 @@ def allowednewrequirements(repo): requirements.REVLOGV1_REQUIREMENT, requirements.REVLOGV2_REQUIREMENT, requirements.CHANGELOGV2_REQUIREMENT, + requirements.DIRSTATE_V2_REQUIREMENT, } for name in compression.compengines: engine = compression.compengines[name] diff --git a/mercurial/upgrade_utils/engine.py b/mercurial/upgrade_utils/engine.py --- a/mercurial/upgrade_utils/engine.py +++ b/mercurial/upgrade_utils/engine.py @@ -30,6 +30,7 @@ from ..revlogutils import ( nodemap, sidedata as sidedatamod, ) +from . import actions as upgrade_actions def get_sidedata_helpers(srcrepo, dstrepo): @@ -458,6 +459,19 @@ def upgrade(ui, srcrepo, dstrepo, upgrad ) ) + if upgrade_actions.dirstatev2 in upgrade_op.upgrade_actions: + ui.status(_(b'upgrading to dirstate-v2 from v1\n')) + upgrade_dirstate(ui, srcrepo, upgrade_op, b'v1', b'v2') + upgrade_op.upgrade_actions.remove(upgrade_actions.dirstatev2) + + if upgrade_actions.dirstatev2 in upgrade_op.removed_actions: + ui.status(_(b'downgrading from dirstate-v2 to v1\n')) + upgrade_dirstate(ui, srcrepo, upgrade_op, b'v2', b'v1') + upgrade_op.removed_actions.remove(upgrade_actions.dirstatev2) + + if not (upgrade_op.upgrade_actions or upgrade_op.removed_actions): + return + if upgrade_op.requirements_only: ui.status(_(b'upgrading repository requirements\n')) scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements) @@ -466,7 +480,7 @@ def upgrade(ui, srcrepo, dstrepo, upgrad # through the whole cloning process elif ( len(upgrade_op.upgrade_actions) == 1 - and b'persistent-nodemap' in upgrade_op._upgrade_actions_names + and b'persistent-nodemap' in upgrade_op.upgrade_actions_names and not upgrade_op.removed_actions ): ui.status( @@ -591,3 +605,28 @@ def upgrade(ui, srcrepo, dstrepo, upgrad backupvfs.unlink(b'store/lock') return backuppath + + +def upgrade_dirstate(ui, srcrepo, upgrade_op, old, new): + if upgrade_op.backup_store: + backuppath = pycompat.mkdtemp( + prefix=b'upgradebackup.', dir=srcrepo.path + ) + ui.status(_(b'replaced files will be backed up at %s\n') % backuppath) + backupvfs = vfsmod.vfs(backuppath) + util.copyfile( + srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires') + ) + util.copyfile( + srcrepo.vfs.join(b'dirstate'), backupvfs.join(b'dirstate') + ) + + assert srcrepo.dirstate._use_dirstate_v2 == (old == b'v2') + srcrepo.dirstate._map._use_dirstate_tree = True + srcrepo.dirstate._map.preload() + srcrepo.dirstate._use_dirstate_v2 = new == b'v2' + srcrepo.dirstate._map._use_dirstate_v2 = srcrepo.dirstate._use_dirstate_v2 + srcrepo.dirstate._dirty = True + srcrepo.dirstate.write(None) + + scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements) diff --git a/tests/test-copies-chain-merge.t b/tests/test-copies-chain-merge.t --- a/tests/test-copies-chain-merge.t +++ b/tests/test-copies-chain-merge.t @@ -1652,6 +1652,7 @@ We upgrade a repository that is not usin $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1691,6 +1692,7 @@ We upgrade a repository that is not usin $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no diff --git a/tests/test-copies-in-changeset.t b/tests/test-copies-in-changeset.t --- a/tests/test-copies-in-changeset.t +++ b/tests/test-copies-in-changeset.t @@ -35,6 +35,7 @@ Check that copies are recorded correctly $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -52,6 +53,7 @@ Check that copies are recorded correctly $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -426,6 +428,7 @@ downgrading $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -456,6 +459,7 @@ downgrading $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -483,6 +487,7 @@ upgrading $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t --- a/tests/test-persistent-nodemap.t +++ b/tests/test-persistent-nodemap.t @@ -57,6 +57,7 @@ As a result, -1 passed from Rust for the $ hg debugformat format-variant repo fncache: yes + dirstate-v2: no dotencode: yes generaldelta: yes share-safe: no @@ -577,6 +578,7 @@ downgrading $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -626,6 +628,7 @@ upgrading $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no diff --git a/tests/test-sidedata.t b/tests/test-sidedata.t --- a/tests/test-sidedata.t +++ b/tests/test-sidedata.t @@ -52,6 +52,7 @@ Check that we can upgrade to sidedata $ hg debugformat -v -R up-no-side-data format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -68,6 +69,7 @@ Check that we can upgrade to sidedata $ hg debugformat -v -R up-no-side-data --config experimental.revlogv2=enable-unstable-format-and-corrupt-my-data format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -90,6 +92,7 @@ Check that we can downgrade from sidedat $ hg debugformat -v -R up-side-data format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -106,6 +109,7 @@ Check that we can downgrade from sidedat $ hg debugformat -v -R up-side-data --config experimental.revlogv2=no format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no diff --git a/tests/test-upgrade-repo.t b/tests/test-upgrade-repo.t --- a/tests/test-upgrade-repo.t +++ b/tests/test-upgrade-repo.t @@ -57,6 +57,7 @@ An upgrade of a repository created with $ hg debugformat format-variant repo fncache: yes + dirstate-v2: no dotencode: yes generaldelta: yes share-safe: no @@ -72,6 +73,7 @@ An upgrade of a repository created with $ hg debugformat --verbose format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -88,6 +90,7 @@ An upgrade of a repository created with $ hg debugformat --verbose --config format.usefncache=no format-variant repo config default fncache: yes no yes + dirstate-v2: no no no dotencode: yes no yes generaldelta: yes yes yes share-safe: no no no @@ -104,6 +107,7 @@ An upgrade of a repository created with $ hg debugformat --verbose --config format.usefncache=no --color=debug format-variant repo config default [formatvariant.name.mismatchconfig|fncache: ][formatvariant.repo.mismatchconfig| yes][formatvariant.config.special| no][formatvariant.default| yes] + [formatvariant.name.uptodate|dirstate-v2: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] [formatvariant.name.mismatchconfig|dotencode: ][formatvariant.repo.mismatchconfig| yes][formatvariant.config.special| no][formatvariant.default| yes] [formatvariant.name.uptodate|generaldelta: ][formatvariant.repo.uptodate| yes][formatvariant.config.default| yes][formatvariant.default| yes] [formatvariant.name.uptodate|share-safe: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] @@ -126,6 +130,12 @@ An upgrade of a repository created with "repo": true }, { + "config": false, + "default": false, + "name": "dirstate-v2", + "repo": false + }, + { "config": true, "default": true, "name": "dotencode", @@ -327,6 +337,7 @@ Various sub-optimal detections work $ hg debugformat format-variant repo fncache: no + dirstate-v2: no dotencode: no generaldelta: no share-safe: no @@ -341,6 +352,7 @@ Various sub-optimal detections work $ hg debugformat --verbose format-variant repo config default fncache: no yes yes + dirstate-v2: no no no dotencode: no yes yes generaldelta: no yes yes share-safe: no no no @@ -357,6 +369,7 @@ Various sub-optimal detections work $ hg debugformat --verbose --config format.usegeneraldelta=no format-variant repo config default fncache: no yes yes + dirstate-v2: no no no dotencode: no yes yes generaldelta: no no yes share-safe: no no no @@ -373,6 +386,7 @@ Various sub-optimal detections work $ hg debugformat --verbose --config format.usegeneraldelta=no --color=debug format-variant repo config default [formatvariant.name.mismatchconfig|fncache: ][formatvariant.repo.mismatchconfig| no][formatvariant.config.default| yes][formatvariant.default| yes] + [formatvariant.name.uptodate|dirstate-v2: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] [formatvariant.name.mismatchconfig|dotencode: ][formatvariant.repo.mismatchconfig| no][formatvariant.config.default| yes][formatvariant.default| yes] [formatvariant.name.mismatchdefault|generaldelta: ][formatvariant.repo.mismatchdefault| no][formatvariant.config.special| no][formatvariant.default| yes] [formatvariant.name.uptodate|share-safe: ][formatvariant.repo.uptodate| no][formatvariant.config.default| no][formatvariant.default| no] @@ -1355,6 +1369,7 @@ upgrade $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1396,6 +1411,7 @@ downgrade $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1440,6 +1456,7 @@ upgrade from hgrc $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1490,6 +1507,7 @@ upgrade $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1537,6 +1555,7 @@ downgrade $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1585,6 +1604,7 @@ upgrade from hgrc $ hg debugformat -v format-variant repo config default fncache: yes yes yes + dirstate-v2: no no no dotencode: yes yes yes generaldelta: yes yes yes share-safe: no no no @@ -1613,3 +1633,105 @@ Demonstrate that nothing to perform upgr $ hg debugupgraderepo --run nothing to do + +#if rust + +Upgrade to dirstate-v2 + + $ hg debugformat -v --config format.exp-dirstate-v2=1 + format-variant repo config default + fncache: yes yes yes + dirstate-v2: no yes no + dotencode: yes yes yes + generaldelta: yes yes yes + share-safe: no no no + sparserevlog: yes yes yes + persistent-nodemap: yes yes no + copies-sdc: no no no + revlog-v2: yes yes no + changelog-v2: no no no + plain-cl-delta: yes yes yes + compression: zstd zstd zstd + compression-level: default default default + $ hg debugupgraderepo --config format.exp-dirstate-v2=1 --run + upgrade will perform the following actions: + + requirements + preserved: dotencode, exp-revlogv2.2, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, sparserevlog, store + added: exp-dirstate-v2 + + dirstate-v2 + "hg status" will be faster + + processed revlogs: + - all-filelogs + - changelog + - manifest + + beginning upgrade... + repository locked and read-only + creating temporary repository to stage upgraded data: $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + (it is safe to interrupt this process any time before data migration completes) + upgrading to dirstate-v2 from v1 + replaced files will be backed up at $TESTTMP/sparserevlogrepo/.hg/upgradebackup.* (glob) + removing temporary repository $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + $ ls .hg/upgradebackup.*/dirstate + .hg/upgradebackup.*/dirstate (glob) + $ hg debugformat -v + format-variant repo config default + fncache: yes yes yes + dirstate-v2: yes no no + dotencode: yes yes yes + generaldelta: yes yes yes + share-safe: no no no + sparserevlog: yes yes yes + persistent-nodemap: yes yes no + copies-sdc: no no no + revlog-v2: yes yes no + changelog-v2: no no no + plain-cl-delta: yes yes yes + compression: zstd zstd zstd + compression-level: default default default + $ hg status + $ dd status=none bs=12 count=1 if=.hg/dirstate + dirstate-v2 + +Downgrade from dirstate-v2 + + $ hg debugupgraderepo --run + upgrade will perform the following actions: + + requirements + preserved: dotencode, exp-revlogv2.2, fncache, generaldelta, persistent-nodemap, revlog-compression-zstd, sparserevlog, store + removed: exp-dirstate-v2 + + processed revlogs: + - all-filelogs + - changelog + - manifest + + beginning upgrade... + repository locked and read-only + creating temporary repository to stage upgraded data: $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + (it is safe to interrupt this process any time before data migration completes) + downgrading from dirstate-v2 to v1 + replaced files will be backed up at $TESTTMP/sparserevlogrepo/.hg/upgradebackup.* (glob) + removing temporary repository $TESTTMP/sparserevlogrepo/.hg/upgrade.* (glob) + $ hg debugformat -v + format-variant repo config default + fncache: yes yes yes + dirstate-v2: no no no + dotencode: yes yes yes + generaldelta: yes yes yes + share-safe: no no no + sparserevlog: yes yes yes + persistent-nodemap: yes yes no + copies-sdc: no no no + revlog-v2: yes yes no + changelog-v2: no no no + plain-cl-delta: yes yes yes + compression: zstd zstd zstd + compression-level: default default default + $ hg status + +#endif