# HG changeset patch # User Arseniy Alekseyev # Date 2022-10-05 19:45:05 # Node ID 52464a20add05debf3618679f3a82d79a9a3b6e8 # Parent 3854eb90629b794bac4d5da93b8a67ba41317cd3 rhg: parallellize computation of [unsure_is_modified] [unsure_is_modified] is called for every file for which we can't determine its status based on its size and mtime alone. In particular, this happens if the mtime of the file changes without its contents changing. Parallellizing this improves performance significantly when we have many of these files. Here's an example run (on a repo with ~400k files after dropping FS caches) ``` before: real 0m53.901s user 0m27.806s sys 0m31.325s after: real 0m32.017s user 0m34.277s sys 1m26.250s ``` Another example run (a different FS): ``` before: real 3m28.479s user 0m31.800s sys 0m25.324s after: real 0m29.751s user 0m41.814s sys 1m15.387s ``` diff --git a/rust/Cargo.lock b/rust/Cargo.lock --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -981,6 +981,7 @@ dependencies = [ "lazy_static", "log", "micro-timer", + "rayon", "regex", "users", "which", diff --git a/rust/hg-core/src/revlog/filelog.rs b/rust/hg-core/src/revlog/filelog.rs --- a/rust/hg-core/src/revlog/filelog.rs +++ b/rust/hg-core/src/revlog/filelog.rs @@ -17,18 +17,21 @@ pub struct Filelog { } impl Filelog { - pub fn open(repo: &Repo, file_path: &HgPath) -> Result { + pub fn open_vfs( + store_vfs: &crate::vfs::Vfs<'_>, + file_path: &HgPath, + ) -> Result { let index_path = store_path(file_path, b".i"); let data_path = store_path(file_path, b".d"); - let revlog = Revlog::open( - &repo.store_vfs(), - index_path, - Some(&data_path), - false, - )?; + let revlog = + Revlog::open(store_vfs, index_path, Some(&data_path), false)?; Ok(Self { revlog }) } + pub fn open(repo: &Repo, file_path: &HgPath) -> Result { + Self::open_vfs(&repo.store_vfs(), file_path) + } + /// The given node ID is that of the file as found in a filelog, not of a /// changeset. pub fn data_for_node( diff --git a/rust/hg-core/src/sparse.rs b/rust/hg-core/src/sparse.rs --- a/rust/hg-core/src/sparse.rs +++ b/rust/hg-core/src/sparse.rs @@ -97,7 +97,7 @@ pub(crate) fn parse_config( Includes, Excludes, None, - }; + } let mut current = Current::None; let mut in_section = false; diff --git a/rust/rhg/Cargo.toml b/rust/rhg/Cargo.toml --- a/rust/rhg/Cargo.toml +++ b/rust/rhg/Cargo.toml @@ -22,3 +22,4 @@ env_logger = "0.9.0" format-bytes = "0.3.0" users = "0.11.0" which = "4.2.5" +rayon = "1.5.1" diff --git a/rust/rhg/src/commands/status.rs b/rust/rhg/src/commands/status.rs --- a/rust/rhg/src/commands/status.rs +++ b/rust/rhg/src/commands/status.rs @@ -29,6 +29,7 @@ use hg::StatusError; use hg::StatusOptions; use hg::{self, narrow, sparse}; use log::info; +use rayon::prelude::*; use std::io; use std::path::PathBuf; @@ -291,16 +292,31 @@ pub fn run(invocation: &crate::CliInvoca let manifest = repo.manifest_for_node(p1).map_err(|e| { CommandError::from((e, &*format!("{:x}", p1.short()))) })?; - for to_check in ds_status.unsure { - if unsure_is_modified(repo, &manifest, &to_check.path)? { + let working_directory_vfs = repo.working_directory_vfs(); + let store_vfs = repo.store_vfs(); + let res: Vec<_> = ds_status + .unsure + .into_par_iter() + .map(|to_check| { + unsure_is_modified( + working_directory_vfs, + store_vfs, + &manifest, + &to_check.path, + ) + .map(|modified| (to_check, modified)) + }) + .collect::>()?; + for (status_path, is_modified) in res.into_iter() { + if is_modified { if display_states.modified { - ds_status.modified.push(to_check); + ds_status.modified.push(status_path); } } else { if display_states.clean { - ds_status.clean.push(to_check.clone()); + ds_status.clean.push(status_path.clone()); } - fixup.push(to_check.path.into_owned()) + fixup.push(status_path.path.into_owned()) } } } @@ -525,11 +541,12 @@ impl DisplayStatusPaths<'_> { /// This meant to be used for those that the dirstate cannot resolve, due /// to time resolution limits. fn unsure_is_modified( - repo: &Repo, + working_directory_vfs: hg::vfs::Vfs, + store_vfs: hg::vfs::Vfs, manifest: &Manifest, hg_path: &HgPath, ) -> Result { - let vfs = repo.working_directory_vfs(); + let vfs = working_directory_vfs; let fs_path = hg_path_to_path_buf(hg_path).expect("HgPath conversion"); let fs_metadata = vfs.symlink_metadata(&fs_path)?; let is_symlink = fs_metadata.file_type().is_symlink(); @@ -549,7 +566,7 @@ fn unsure_is_modified( if entry.flags != fs_flags { return Ok(true); } - let filelog = repo.filelog(hg_path)?; + let filelog = hg::filelog::Filelog::open_vfs(&store_vfs, hg_path)?; let fs_len = fs_metadata.len(); let file_node = entry.node_id()?; let filelog_entry = filelog.entry_for_node(file_node).map_err(|_| {