rust-pathauditor: use interior mutability for use in multi-threaded contexts

The usual recommendation for using `RwLock` or `Mutex` is that if there are
about as many write as there are reads, use `Mutex`, and if there are more
reads than writes, use `RwLock`.

If after the main bottleneck (i.e. parallel traversal) is removed this shows
up on profiles, we should investigate using the `parking_lot` since we don't
need a poisoning API, or maybe move to different types of caches entirely.

Differential Revision: https://phab.mercurial-scm.org/D8213

rust/hg-core/src/dirstate/status.rs

@@ -1,770 +1,770 b''
 // status.rs
 //
 // Copyright 2019 Raphaël Gomès <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Rust implementation of dirstate.status (dirstate.py).
 //! It is currently missing a lot of functionality compared to the Python one
 //! and will only be triggered in narrow cases.
 
 use crate::{
     dirstate::SIZE_FROM_OTHER_PARENT,
     filepatterns::PatternFileWarning,
     matchers::{get_ignore_function, Matcher, VisitChildrenSet},
     utils::{
         files::{find_dirs, HgMetadata},
         hg_path::{
             hg_path_to_path_buf, os_string_to_hg_path_buf, HgPath, HgPathBuf,
             HgPathError,
         },
         path_auditor::PathAuditor,
     },
     CopyMap, DirstateEntry, DirstateMap, EntryState, FastHashMap,
     PatternError,
 };
 use lazy_static::lazy_static;
 use rayon::prelude::*;
 use std::collections::VecDeque;
 use std::{
     borrow::Cow,
     collections::HashSet,
     fs::{read_dir, DirEntry},
     io::ErrorKind,
     ops::Deref,
     path::Path,
 };
 
 /// Wrong type of file from a `BadMatch`
 /// Note: a lot of those don't exist on all platforms.
 #[derive(Debug)]
 pub enum BadType {
     CharacterDevice,
     BlockDevice,
     FIFO,
     Socket,
     Directory,
     Unknown,
 }
 
 impl ToString for BadType {
     fn to_string(&self) -> String {
         match self {
             BadType::CharacterDevice => "character device",
             BadType::BlockDevice => "block device",
             BadType::FIFO => "fifo",
             BadType::Socket => "socket",
             BadType::Directory => "directory",
             BadType::Unknown => "unknown",
         }
         .to_string()
     }
 }
 
 /// Was explicitly matched but cannot be found/accessed
 #[derive(Debug)]
 pub enum BadMatch {
     OsError(i32),
     BadType(BadType),
 }
 
 /// Marker enum used to dispatch new status entries into the right collections.
 /// Is similar to `crate::EntryState`, but represents the transient state of
 /// entries during the lifetime of a command.
 #[derive(Debug)]
 enum Dispatch {
     Unsure,
     Modified,
     Added,
     Removed,
     Deleted,
     Clean,
     Unknown,
     Ignored,
     /// Empty dispatch, the file is not worth listing
     None,
     /// Was explicitly matched but cannot be found/accessed
     Bad(BadMatch),
     Directory {
         /// True if the directory used to be a file in the dmap so we can say
         /// that it's been removed.
         was_file: bool,
     },
 }
 
 type IoResult<T> = std::io::Result<T>;
 
 /// Dates and times that are outside the 31-bit signed range are compared
 /// modulo 2^31. This should prevent hg from behaving badly with very large
 /// files or corrupt dates while still having a high probability of detecting
 /// changes. (issue2608)
 /// TODO I haven't found a way of having `b` be `Into<i32>`, since `From<u64>`
 /// is not defined for `i32`, and there is no `As` trait. This forces the
 /// caller to cast `b` as `i32`.
 fn mod_compare(a: i32, b: i32) -> bool {
     a & i32::max_value() != b & i32::max_value()
 }
 
 /// Return a sorted list containing information about the entries
 /// in the directory.
 ///
 /// * `skip_dot_hg` - Return an empty vec if `path` contains a `.hg` directory
 fn list_directory(
     path: impl AsRef<Path>,
     skip_dot_hg: bool,
 ) -> std::io::Result<Vec<(HgPathBuf, DirEntry)>> {
     let mut results = vec![];
     let entries = read_dir(path.as_ref())?;
 
     for entry in entries {
         let entry = entry?;
         let filename = os_string_to_hg_path_buf(entry.file_name())?;
         let file_type = entry.file_type()?;
         if skip_dot_hg && filename.as_bytes() == b".hg" && file_type.is_dir() {
             return Ok(vec![]);
         } else {
             results.push((HgPathBuf::from(filename), entry))
         }
     }
 
     results.sort_unstable_by_key(|e| e.0.clone());
     Ok(results)
 }
 
 /// The file corresponding to the dirstate entry was found on the filesystem.
 fn dispatch_found(
     filename: impl AsRef<HgPath>,
     entry: DirstateEntry,
     metadata: HgMetadata,
     copy_map: &CopyMap,
     options: StatusOptions,
 ) -> Dispatch {
     let DirstateEntry {
         state,
         mode,
         mtime,
         size,
     } = entry;
 
     let HgMetadata {
         st_mode,
         st_size,
         st_mtime,
         ..
     } = metadata;
 
     match state {
         EntryState::Normal => {
             let size_changed = mod_compare(size, st_size as i32);
             let mode_changed =
                 (mode ^ st_mode as i32) & 0o100 != 0o000 && options.check_exec;
             let metadata_changed = size >= 0 && (size_changed || mode_changed);
             let other_parent = size == SIZE_FROM_OTHER_PARENT;
             if metadata_changed
                 || other_parent
                 || copy_map.contains_key(filename.as_ref())
             {
                 Dispatch::Modified
             } else if mod_compare(mtime, st_mtime as i32) {
                 Dispatch::Unsure
             } else if st_mtime == options.last_normal_time {
                 // the file may have just been marked as normal and
                 // it may have changed in the same second without
                 // changing its size. This can happen if we quickly
                 // do multiple commits. Force lookup, so we don't
                 // miss such a racy file change.
                 Dispatch::Unsure
             } else if options.list_clean {
                 Dispatch::Clean
             } else {
                 Dispatch::None
             }
         }
         EntryState::Merged => Dispatch::Modified,
         EntryState::Added => Dispatch::Added,
         EntryState::Removed => Dispatch::Removed,
         EntryState::Unknown => Dispatch::Unknown,
     }
 }
 
 /// The file corresponding to this Dirstate entry is missing.
 fn dispatch_missing(state: EntryState) -> Dispatch {
     match state {
         // File was removed from the filesystem during commands
         EntryState::Normal | EntryState::Merged | EntryState::Added => {
             Dispatch::Deleted
         }
         // File was removed, everything is normal
         EntryState::Removed => Dispatch::Removed,
         // File is unknown to Mercurial, everything is normal
         EntryState::Unknown => Dispatch::Unknown,
     }
 }
 
 lazy_static! {
     static ref DEFAULT_WORK: HashSet<&'static HgPath> = {
         let mut h = HashSet::new();
         h.insert(HgPath::new(b""));
         h
     };
 }
 
 /// Get stat data about the files explicitly specified by match.
 /// TODO subrepos
 fn walk_explicit<'a>(
     files: Option<&'a HashSet<&HgPath>>,
     dmap: &'a DirstateMap,
     root_dir: impl AsRef<Path> + Sync + Send + 'a,
     options: StatusOptions,
 ) -> impl ParallelIterator<Item = IoResult<(&'a HgPath, Dispatch)>> {
     files
         .unwrap_or(&DEFAULT_WORK)
         .par_iter()
         .map(move |filename| {
             // TODO normalization
             let normalized = filename.as_ref();
 
             let buf = match hg_path_to_path_buf(normalized) {
                 Ok(x) => x,
                 Err(e) => return Some(Err(e.into())),
             };
             let target = root_dir.as_ref().join(buf);
             let st = target.symlink_metadata();
             let in_dmap = dmap.get(normalized);
             match st {
                 Ok(meta) => {
                     let file_type = meta.file_type();
                     return if file_type.is_file() || file_type.is_symlink() {
                         if let Some(entry) = in_dmap {
                             return Some(Ok((
                                 normalized,
                                 dispatch_found(
                                     &normalized,
                                     *entry,
                                     HgMetadata::from_metadata(meta),
                                     &dmap.copy_map,
                                     options,
                                 ),
                             )));
                         }
                         Some(Ok((normalized, Dispatch::Unknown)))
                     } else {
                         if file_type.is_dir() {
                             Some(Ok((
                                 normalized,
                                 Dispatch::Directory {
                                     was_file: in_dmap.is_some(),
                                 },
                             )))
                         } else {
                             Some(Ok((
                                 normalized,
                                 Dispatch::Bad(BadMatch::BadType(
                                     // TODO do more than unknown
                                     // Support for all `BadType` variant
                                     // varies greatly between platforms.
                                     // So far, no tests check the type and
                                     // this should be good enough for most
                                     // users.
                                     BadType::Unknown,
                                 )),
                             )))
                         }
                     };
                 }
                 Err(_) => {
                     if let Some(entry) = in_dmap {
                         return Some(Ok((
                             normalized,
                             dispatch_missing(entry.state),
                         )));
                     }
                 }
             };
             None
         })
         .flatten()
 }
 
 #[derive(Debug, Copy, Clone)]
 pub struct StatusOptions {
     /// Remember the most recent modification timeslot for status, to make
     /// sure we won't miss future size-preserving file content modifications
     /// that happen within the same timeslot.
     pub last_normal_time: i64,
     /// Whether we are on a filesystem with UNIX-like exec flags
     pub check_exec: bool,
     pub list_clean: bool,
     pub list_unknown: bool,
     pub list_ignored: bool,
 }
 
 /// Dispatch a single file found during `traverse`.
 /// If `file` is a folder that needs to be traversed, it will be pushed into
 /// `work`.
 fn traverse_worker<'a>(
     work: &mut VecDeque<HgPathBuf>,
     matcher: &impl Matcher,
     dmap: &DirstateMap,
     filename: impl AsRef<HgPath>,
     dir_entry: &DirEntry,
     ignore_fn: &impl for<'r> Fn(&'r HgPath) -> bool,
     dir_ignore_fn: &impl for<'r> Fn(&'r HgPath) -> bool,
     options: StatusOptions,
 ) -> Option<IoResult<(Cow<'a, HgPath>, Dispatch)>> {
     let file_type = match dir_entry.file_type() {
         Ok(x) => x,
         Err(e) => return Some(Err(e.into())),
     };
     let filename = filename.as_ref();
     let entry_option = dmap.get(filename);
 
     if file_type.is_dir() {
         // Do we need to traverse it?
         if !ignore_fn(&filename) || options.list_ignored {
             work.push_front(filename.to_owned());
         }
         // Nested `if` until `rust-lang/rust#53668` is stable
         if let Some(entry) = entry_option {
             // Used to be a file, is now a folder
             if matcher.matches_everything() || matcher.matches(&filename) {
                 return Some(Ok((
                     Cow::Owned(filename.to_owned()),
                     dispatch_missing(entry.state),
                 )));
             }
         }
     } else if file_type.is_file() || file_type.is_symlink() {
         if let Some(entry) = entry_option {
             if matcher.matches_everything() || matcher.matches(&filename) {
                 let metadata = match dir_entry.metadata() {
                     Ok(x) => x,
                     Err(e) => return Some(Err(e.into())),
                 };
                 return Some(Ok((
                     Cow::Owned(filename.to_owned()),
                     dispatch_found(
                         &filename,
                         *entry,
                         HgMetadata::from_metadata(metadata),
                         &dmap.copy_map,
                         options,
                     ),
                 )));
             }
         } else if (matcher.matches_everything() || matcher.matches(&filename))
             && !ignore_fn(&filename)
         {
             if (options.list_ignored || matcher.exact_match(&filename))
                 && dir_ignore_fn(&filename)
             {
                 if options.list_ignored {
                     return Some(Ok((
                         Cow::Owned(filename.to_owned()),
                         Dispatch::Ignored,
                     )));
                 }
             } else {
                 return Some(Ok((
                     Cow::Owned(filename.to_owned()),
                     Dispatch::Unknown,
                 )));
             }
         }
     } else if let Some(entry) = entry_option {
         // Used to be a file or a folder, now something else.
         if matcher.matches_everything() || matcher.matches(&filename) {
             return Some(Ok((
                 Cow::Owned(filename.to_owned()),
                 dispatch_missing(entry.state),
             )));
         }
     }
     None
 }
 
 /// Walk the working directory recursively to look for changes compared to the
 /// current `DirstateMap`.
 fn traverse<'a>(
     matcher: &(impl Matcher + Sync),
     root_dir: impl AsRef<Path>,
     dmap: &DirstateMap,
     path: impl AsRef<HgPath>,
     old_results: FastHashMap<Cow<'a, HgPath>, Dispatch>,
     ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
     dir_ignore_fn: &(impl for<'r> Fn(&'r HgPath) -> bool + Sync),
     options: StatusOptions,
 ) -> IoResult<FastHashMap<Cow<'a, HgPath>, Dispatch>> {
     let root_dir = root_dir.as_ref();
     let mut new_results = FastHashMap::default();
 
     let mut work = VecDeque::new();
     work.push_front(path.as_ref().to_owned());
 
     while let Some(ref directory) = work.pop_front() {
         if directory.as_bytes() == b".hg" {
             continue;
         }
         let visit_entries = match matcher.visit_children_set(directory) {
             VisitChildrenSet::Empty => continue,
             VisitChildrenSet::This | VisitChildrenSet::Recursive => None,
             VisitChildrenSet::Set(set) => Some(set),
         };
         let buf = hg_path_to_path_buf(directory)?;
         let dir_path = root_dir.join(buf);
 
         let skip_dot_hg = !directory.as_bytes().is_empty();
         let entries = match list_directory(dir_path, skip_dot_hg) {
             Err(e) => match e.kind() {
                 ErrorKind::NotFound | ErrorKind::PermissionDenied => {
                     new_results.insert(
                         Cow::Owned(directory.to_owned()),
                         Dispatch::Bad(BadMatch::OsError(
                             // Unwrapping here is OK because the error always
                             // is a real os error
                             e.raw_os_error().unwrap(),
                         )),
                     );
                     continue;
                 }
                 _ => return Err(e),
             },
             Ok(entries) => entries,
         };
 
         for (filename, dir_entry) in entries {
             if let Some(ref set) = visit_entries {
                 if !set.contains(filename.deref()) {
                     continue;
                 }
             }
             // TODO normalize
             let filename = if directory.is_empty() {
                 filename.to_owned()
             } else {
                 directory.join(&filename)
             };
 
             if !old_results.contains_key(filename.deref()) {
                 if let Some((res, dispatch)) = traverse_worker(
                     &mut work,
                     matcher,
                     &dmap,
                     &filename,
                     &dir_entry,
                     &ignore_fn,
                     &dir_ignore_fn,
                     options,
                 )
                 .transpose()?
                 {
                     new_results.insert(res, dispatch);
                 }
             }
         }
     }
 
     new_results.extend(old_results.into_iter());
 
     Ok(new_results)
 }
 
 /// Stat all entries in the `DirstateMap` and mark them for dispatch.
 fn stat_dmap_entries(
     dmap: &DirstateMap,
     root_dir: impl AsRef<Path> + Sync + Send,
     options: StatusOptions,
 ) -> impl ParallelIterator<Item = IoResult<(&HgPath, Dispatch)>> {
     dmap.par_iter().map(move |(filename, entry)| {
         let filename: &HgPath = filename;
         let filename_as_path = hg_path_to_path_buf(filename)?;
         let meta = root_dir.as_ref().join(filename_as_path).symlink_metadata();
 
         match meta {
             Ok(ref m)
                 if !(m.file_type().is_file()
                     || m.file_type().is_symlink()) =>
             {
                 Ok((filename, dispatch_missing(entry.state)))
             }
             Ok(m) => Ok((
                 filename,
                 dispatch_found(
                     filename,
                     *entry,
                     HgMetadata::from_metadata(m),
                     &dmap.copy_map,
                     options,
                 ),
             )),
             Err(ref e)
                 if e.kind() == ErrorKind::NotFound
                     || e.raw_os_error() == Some(20) =>
             {
                 // Rust does not yet have an `ErrorKind` for
                 // `NotADirectory` (errno 20)
                 // It happens if the dirstate contains `foo/bar` and
                 // foo is not a directory
                 Ok((filename, dispatch_missing(entry.state)))
             }
             Err(e) => Err(e),
         }
     })
 }
 
 pub struct DirstateStatus<'a> {
     pub modified: Vec<Cow<'a, HgPath>>,
     pub added: Vec<Cow<'a, HgPath>>,
     pub removed: Vec<Cow<'a, HgPath>>,
     pub deleted: Vec<Cow<'a, HgPath>>,
     pub clean: Vec<Cow<'a, HgPath>>,
     pub ignored: Vec<Cow<'a, HgPath>>,
     pub unknown: Vec<Cow<'a, HgPath>>,
     pub bad: Vec<(Cow<'a, HgPath>, BadMatch)>,
 }
 
 fn build_response<'a>(
     results: impl IntoIterator<Item = (Cow<'a, HgPath>, Dispatch)>,
 ) -> (Vec<Cow<'a, HgPath>>, DirstateStatus<'a>) {
     let mut lookup = vec![];
     let mut modified = vec![];
     let mut added = vec![];
     let mut removed = vec![];
     let mut deleted = vec![];
     let mut clean = vec![];
     let mut ignored = vec![];
     let mut unknown = vec![];
     let mut bad = vec![];
 
     for (filename, dispatch) in results.into_iter() {
         match dispatch {
             Dispatch::Unknown => unknown.push(filename),
             Dispatch::Unsure => lookup.push(filename),
             Dispatch::Modified => modified.push(filename),
             Dispatch::Added => added.push(filename),
             Dispatch::Removed => removed.push(filename),
             Dispatch::Deleted => deleted.push(filename),
             Dispatch::Clean => clean.push(filename),
             Dispatch::Ignored => ignored.push(filename),
             Dispatch::None => {}
             Dispatch::Bad(reason) => bad.push((filename, reason)),
             Dispatch::Directory { .. } => {}
         }
     }
 
     (
         lookup,
         DirstateStatus {
             modified,
             added,
             removed,
             deleted,
             clean,
             ignored,
             unknown,
             bad,
         },
     )
 }
 
 pub enum StatusError {
     IO(std::io::Error),
     Path(HgPathError),
     Pattern(PatternError),
 }
 
 pub type StatusResult<T> = Result<T, StatusError>;
 
 impl From<PatternError> for StatusError {
     fn from(e: PatternError) -> Self {
         StatusError::Pattern(e)
     }
 }
 impl From<HgPathError> for StatusError {
     fn from(e: HgPathError) -> Self {
         StatusError::Path(e)
     }
 }
 impl From<std::io::Error> for StatusError {
     fn from(e: std::io::Error) -> Self {
         StatusError::IO(e)
     }
 }
 
 impl ToString for StatusError {
     fn to_string(&self) -> String {
         match self {
             StatusError::IO(e) => e.to_string(),
             StatusError::Path(e) => e.to_string(),
             StatusError::Pattern(e) => e.to_string(),
         }
     }
 }
 
 /// Get the status of files in the working directory.
 ///
 /// This is the current entry-point for `hg-core` and is realistically unusable
 /// outside of a Python context because its arguments need to provide a lot of
 /// information that will not be necessary in the future.
 pub fn status<'a: 'c, 'b: 'c, 'c>(
     dmap: &'a DirstateMap,
     matcher: &'b (impl Matcher + Sync),
     root_dir: impl AsRef<Path> + Sync + Send + Copy + 'c,
     ignore_files: &[impl AsRef<Path> + 'c],
     options: StatusOptions,
 ) -> StatusResult<(
     (Vec<Cow<'c, HgPath>>, DirstateStatus<'c>),
     Vec<PatternFileWarning>,
 )> {
     let (ignore_fn, warnings) = get_ignore_function(&ignore_files, root_dir)?;
 
     // Is the path or one of its ancestors ignored?
     let dir_ignore_fn = |dir: &_| {
         if ignore_fn(dir) {
             true
         } else {
             for p in find_dirs(dir) {
                 if ignore_fn(p) {
                     return true;
                 }
             }
             false
         }
     };
 
     let files = matcher.file_set();
 
     // Step 1: check the files explicitly mentioned by the user
     let explicit = walk_explicit(files, &dmap, root_dir, options);
     let (work, mut results): (Vec<_>, FastHashMap<_, _>) = explicit
         .filter_map(Result::ok)
         .map(|(filename, dispatch)| (Cow::Borrowed(filename), dispatch))
         .partition(|(_, dispatch)| match dispatch {
             Dispatch::Directory { .. } => true,
             _ => false,
         });
 
     // Step 2: recursively check the working directory for changes if needed
     for (dir, dispatch) in work {
         match dispatch {
             Dispatch::Directory { was_file } => {
                 if was_file {
                     results.insert(dir.to_owned(), Dispatch::Removed);
                 }
                 if options.list_ignored
                     || options.list_unknown && !dir_ignore_fn(&dir)
                 {
                     results = traverse(
                         matcher,
                         root_dir,
                         &dmap,
                         &dir,
                         results,
                         &ignore_fn,
                         &dir_ignore_fn,
                         options,
                     )?;
                 }
             }
             _ => unreachable!("There can only be directories in `work`"),
         }
     }
 
     if !matcher.is_exact() {
         // Step 3: Check the remaining files from the dmap.
         // If a dmap file is not in results yet, it was either
         // a) not matched b) ignored, c) missing, or d) under a
         // symlink directory.
 
         if options.list_unknown {
             let to_visit: Box<dyn Iterator<Item = (&HgPath, &DirstateEntry)>> =
                 if results.is_empty() && matcher.matches_everything() {
                     Box::new(dmap.iter().map(|(f, e)| (f.deref(), e)))
                 } else {
                     Box::new(dmap.iter().filter_map(|(f, e)| {
                         if !results.contains_key(f.deref())
                             && matcher.matches(f)
                         {
                             Some((f.deref(), e))
                         } else {
                             None
                         }
                     }))
                 };
             let mut to_visit: Vec<_> = to_visit.collect();
             to_visit.sort_by(|a, b| a.0.cmp(&b.0));
 
             // We walked all dirs under the roots that weren't ignored, and
             // everything that matched was stat'ed and is already in results.
             // The rest must thus be ignored or under a symlink.
-            let mut path_auditor = PathAuditor::new(root_dir);
+            let path_auditor = PathAuditor::new(root_dir);
 
             for (ref filename, entry) in to_visit {
                 // Report ignored items in the dmap as long as they are not
                 // under a symlink directory.
                 if path_auditor.check(filename) {
                     // TODO normalize for case-insensitive filesystems
                     let buf = hg_path_to_path_buf(filename)?;
                     results.insert(
                         Cow::Borrowed(filename),
                         match root_dir.as_ref().join(&buf).symlink_metadata() {
                             // File was just ignored, no links, and exists
                             Ok(meta) => {
                                 let metadata = HgMetadata::from_metadata(meta);
                                 dispatch_found(
                                     filename,
                                     *entry,
                                     metadata,
                                     &dmap.copy_map,
                                     options,
                                 )
                             }
                             // File doesn't exist
                             Err(_) => dispatch_missing(entry.state),
                         },
                     );
                 } else {
                     // It's either missing or under a symlink directory which
                     // we, in this case, report as missing.
                     results.insert(
                         Cow::Borrowed(filename),
                         dispatch_missing(entry.state),
                     );
                 }
             }
         } else {
             // We may not have walked the full directory tree above, so stat
             // and check everything we missed.
             let stat_results = stat_dmap_entries(&dmap, root_dir, options);
             results.par_extend(stat_results.flatten().map(
                 |(filename, dispatch)| (Cow::Borrowed(filename), dispatch),
             ));
         }
     }
 
     let results = results.into_iter().filter_map(|(filename, dispatch)| {
         match dispatch {
             Dispatch::Bad(_) => return Some((filename, dispatch)),
             _ => {}
         };
         // TODO do this in //, not at the end
         if !dmap.contains_key(filename.deref()) {
             if (options.list_ignored || matcher.exact_match(&filename))
                 && dir_ignore_fn(&filename)
             {
                 if options.list_ignored {
                     return Some((filename.to_owned(), Dispatch::Ignored));
                 }
             } else {
                 if !ignore_fn(&filename) {
                     return Some((filename.to_owned(), Dispatch::Unknown));
                 }
             }
             return None;
         }
         Some((filename, dispatch))
     });
 
     Ok((build_response(results), warnings))
 }

rust/hg-core/src/utils/files.rs

@@ -1,384 +1,384 b''
 // files.rs
 //
 // Copyright 2019
 // Raphaël Gomès <[e-mail]>,
 // Yuya Nishihara <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Functions for fiddling with files.
 
 use crate::utils::{
     hg_path::{path_to_hg_path_buf, HgPath, HgPathBuf, HgPathError},
     path_auditor::PathAuditor,
     replace_slice,
 };
 use lazy_static::lazy_static;
 use same_file::is_same_file;
 use std::borrow::ToOwned;
 use std::fs::Metadata;
 use std::iter::FusedIterator;
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 
 pub fn get_path_from_bytes(bytes: &[u8]) -> &Path {
     let os_str;
     #[cfg(unix)]
     {
         use std::os::unix::ffi::OsStrExt;
         os_str = std::ffi::OsStr::from_bytes(bytes);
     }
     // TODO Handle other platforms
     // TODO: convert from WTF8 to Windows MBCS (ANSI encoding).
     // Perhaps, the return type would have to be Result<PathBuf>.
 
     Path::new(os_str)
 }
 
 // TODO: need to convert from WTF8 to MBCS bytes on Windows.
 // that's why Vec<u8> is returned.
 #[cfg(unix)]
 pub fn get_bytes_from_path(path: impl AsRef<Path>) -> Vec<u8> {
     use std::os::unix::ffi::OsStrExt;
     path.as_ref().as_os_str().as_bytes().to_vec()
 }
 
 /// An iterator over repository path yielding itself and its ancestors.
 #[derive(Copy, Clone, Debug)]
 pub struct Ancestors<'a> {
     next: Option<&'a HgPath>,
 }
 
 impl<'a> Iterator for Ancestors<'a> {
     type Item = &'a HgPath;
 
     fn next(&mut self) -> Option<Self::Item> {
         let next = self.next;
         self.next = match self.next {
             Some(s) if s.is_empty() => None,
             Some(s) => {
                 let p = s.bytes().rposition(|c| *c == b'/').unwrap_or(0);
                 Some(HgPath::new(&s.as_bytes()[..p]))
             }
             None => None,
         };
         next
     }
 }
 
 impl<'a> FusedIterator for Ancestors<'a> {}
 
 /// An iterator over repository path yielding itself and its ancestors.
 #[derive(Copy, Clone, Debug)]
 pub(crate) struct AncestorsWithBase<'a> {
     next: Option<(&'a HgPath, &'a HgPath)>,
 }
 
 impl<'a> Iterator for AncestorsWithBase<'a> {
     type Item = (&'a HgPath, &'a HgPath);
 
     fn next(&mut self) -> Option<Self::Item> {
         let next = self.next;
         self.next = match self.next {
             Some((s, _)) if s.is_empty() => None,
             Some((s, _)) => Some(s.split_filename()),
             None => None,
         };
         next
     }
 }
 
 impl<'a> FusedIterator for AncestorsWithBase<'a> {}
 
 /// Returns an iterator yielding ancestor directories of the given repository
 /// path.
 ///
 /// The path is separated by '/', and must not start with '/'.
 ///
 /// The path itself isn't included unless it is b"" (meaning the root
 /// directory.)
 pub fn find_dirs<'a>(path: &'a HgPath) -> Ancestors<'a> {
     let mut dirs = Ancestors { next: Some(path) };
     if !path.is_empty() {
         dirs.next(); // skip itself
     }
     dirs
 }
 
 /// Returns an iterator yielding ancestor directories of the given repository
 /// path.
 ///
 /// The path is separated by '/', and must not start with '/'.
 ///
 /// The path itself isn't included unless it is b"" (meaning the root
 /// directory.)
 pub(crate) fn find_dirs_with_base<'a>(
     path: &'a HgPath,
 ) -> AncestorsWithBase<'a> {
     let mut dirs = AncestorsWithBase {
         next: Some((path, HgPath::new(b""))),
     };
     if !path.is_empty() {
         dirs.next(); // skip itself
     }
     dirs
 }
 
 /// TODO more than ASCII?
 pub fn normalize_case(path: &HgPath) -> HgPathBuf {
     #[cfg(windows)] // NTFS compares via upper()
     return path.to_ascii_uppercase();
     #[cfg(unix)]
     path.to_ascii_lowercase()
 }
 
 lazy_static! {
     static ref IGNORED_CHARS: Vec<Vec<u8>> = {
         [
             0x200c, 0x200d, 0x200e, 0x200f, 0x202a, 0x202b, 0x202c, 0x202d,
             0x202e, 0x206a, 0x206b, 0x206c, 0x206d, 0x206e, 0x206f, 0xfeff,
         ]
         .iter()
         .map(|code| {
             std::char::from_u32(*code)
                 .unwrap()
                 .encode_utf8(&mut [0; 3])
                 .bytes()
                 .collect()
         })
         .collect()
     };
 }
 
 fn hfs_ignore_clean(bytes: &[u8]) -> Vec<u8> {
     let mut buf = bytes.to_owned();
     let needs_escaping = bytes.iter().any(|b| *b == b'\xe2' || *b == b'\xef');
     if needs_escaping {
         for forbidden in IGNORED_CHARS.iter() {
             replace_slice(&mut buf, forbidden, &[])
         }
         buf
     } else {
         buf
     }
 }
 
 pub fn lower_clean(bytes: &[u8]) -> Vec<u8> {
     hfs_ignore_clean(&bytes.to_ascii_lowercase())
 }
 
 #[derive(Eq, PartialEq, Ord, PartialOrd, Copy, Clone)]
 pub struct HgMetadata {
     pub st_dev: u64,
     pub st_mode: u32,
     pub st_nlink: u64,
     pub st_size: u64,
     pub st_mtime: i64,
     pub st_ctime: i64,
 }
 
 // TODO support other plaforms
 #[cfg(unix)]
 impl HgMetadata {
     pub fn from_metadata(metadata: Metadata) -> Self {
         use std::os::unix::fs::MetadataExt;
         Self {
             st_dev: metadata.dev(),
             st_mode: metadata.mode(),
             st_nlink: metadata.nlink(),
             st_size: metadata.size(),
             st_mtime: metadata.mtime(),
             st_ctime: metadata.ctime(),
         }
     }
 }
 
 /// Returns the canonical path of `name`, given `cwd` and `root`
 pub fn canonical_path(
     root: impl AsRef<Path>,
     cwd: impl AsRef<Path>,
     name: impl AsRef<Path>,
 ) -> Result<PathBuf, HgPathError> {
     // TODO add missing normalization for other platforms
     let root = root.as_ref();
     let cwd = cwd.as_ref();
     let name = name.as_ref();
 
     let name = if !name.is_absolute() {
         root.join(&cwd).join(&name)
     } else {
         name.to_owned()
     };
-    let mut auditor = PathAuditor::new(&root);
+    let auditor = PathAuditor::new(&root);
     if name != root && name.starts_with(&root) {
         let name = name.strip_prefix(&root).unwrap();
         auditor.audit_path(path_to_hg_path_buf(name)?)?;
         return Ok(name.to_owned());
     } else if name == root {
         return Ok("".into());
     } else {
         // Determine whether `name' is in the hierarchy at or beneath `root',
         // by iterating name=name.parent() until it returns `None` (can't
         // check name == '/', because that doesn't work on windows).
         let mut name = name.deref();
         let original_name = name.to_owned();
         loop {
             let same = is_same_file(&name, &root).unwrap_or(false);
             if same {
                 if name == original_name {
                     // `name` was actually the same as root (maybe a symlink)
                     return Ok("".into());
                 }
                 // `name` is a symlink to root, so `original_name` is under
                 // root
                 let rel_path = original_name.strip_prefix(&name).unwrap();
                 auditor.audit_path(path_to_hg_path_buf(&rel_path)?)?;
                 return Ok(rel_path.to_owned());
             }
             name = match name.parent() {
                 None => break,
                 Some(p) => p,
             };
         }
         // TODO hint to the user about using --cwd
         // Bubble up the responsibility to Python for now
         Err(HgPathError::NotUnderRoot {
             path: original_name.to_owned(),
             root: root.to_owned(),
         })
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
 
     #[test]
     fn find_dirs_some() {
         let mut dirs = super::find_dirs(HgPath::new(b"foo/bar/baz"));
         assert_eq!(dirs.next(), Some(HgPath::new(b"foo/bar")));
         assert_eq!(dirs.next(), Some(HgPath::new(b"foo")));
         assert_eq!(dirs.next(), Some(HgPath::new(b"")));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn find_dirs_empty() {
         // looks weird, but mercurial.pathutil.finddirs(b"") yields b""
         let mut dirs = super::find_dirs(HgPath::new(b""));
         assert_eq!(dirs.next(), Some(HgPath::new(b"")));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn test_find_dirs_with_base_some() {
         let mut dirs = super::find_dirs_with_base(HgPath::new(b"foo/bar/baz"));
         assert_eq!(
             dirs.next(),
             Some((HgPath::new(b"foo/bar"), HgPath::new(b"baz")))
         );
         assert_eq!(
             dirs.next(),
             Some((HgPath::new(b"foo"), HgPath::new(b"bar")))
         );
         assert_eq!(dirs.next(), Some((HgPath::new(b""), HgPath::new(b"foo"))));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn test_find_dirs_with_base_empty() {
         let mut dirs = super::find_dirs_with_base(HgPath::new(b""));
         assert_eq!(dirs.next(), Some((HgPath::new(b""), HgPath::new(b""))));
         assert_eq!(dirs.next(), None);
         assert_eq!(dirs.next(), None);
     }
 
     #[test]
     fn test_canonical_path() {
         let root = Path::new("/repo");
         let cwd = Path::new("/dir");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Err(HgPathError::NotUnderRoot {
                 path: PathBuf::from("/dir/filename"),
                 root: root.to_path_buf()
             })
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Err(HgPathError::NotUnderRoot {
                 path: PathBuf::from("/filename"),
                 root: root.to_path_buf()
             })
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/");
         let name = Path::new("repo/filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Ok(PathBuf::from("filename"))
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/repo");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Ok(PathBuf::from("filename"))
         );
 
         let root = Path::new("/repo");
         let cwd = Path::new("/repo/subdir");
         let name = Path::new("filename");
         assert_eq!(
             canonical_path(root, cwd, name),
             Ok(PathBuf::from("subdir/filename"))
         );
     }
 
     #[test]
     fn test_canonical_path_not_rooted() {
         use std::fs::create_dir;
         use tempfile::tempdir;
 
         let base_dir = tempdir().unwrap();
         let base_dir_path = base_dir.path();
         let beneath_repo = base_dir_path.join("a");
         let root = base_dir_path.join("a/b");
         let out_of_repo = base_dir_path.join("c");
         let under_repo_symlink = out_of_repo.join("d");
 
         create_dir(&beneath_repo).unwrap();
         create_dir(&root).unwrap();
 
         // TODO make portable
         std::os::unix::fs::symlink(&root, &out_of_repo).unwrap();
 
         assert_eq!(
             canonical_path(&root, Path::new(""), out_of_repo),
             Ok(PathBuf::from(""))
         );
         assert_eq!(
             canonical_path(&root, Path::new(""), &beneath_repo),
             Err(HgPathError::NotUnderRoot {
                 path: beneath_repo.to_owned(),
                 root: root.to_owned()
             })
         );
         assert_eq!(
             canonical_path(&root, Path::new(""), &under_repo_symlink),
             Ok(PathBuf::from("d"))
         );
     }
 }

rust/hg-core/src/utils/path_auditor.rs

@@ -1,231 +1,232 b''
 // path_auditor.rs
 //
 // Copyright 2020
 // Raphaël Gomès <[e-mail]>,
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 use crate::utils::{
     files::lower_clean,
     find_slice_in_slice,
     hg_path::{hg_path_to_path_buf, HgPath, HgPathBuf, HgPathError},
 };
 use std::collections::HashSet;
 use std::path::{Path, PathBuf};
+use std::sync::{Mutex, RwLock};
 
 /// Ensures that a path is valid for use in the repository i.e. does not use
 /// any banned components, does not traverse a symlink, etc.
 #[derive(Debug, Default)]
 pub struct PathAuditor {
-    audited: HashSet<HgPathBuf>,
-    audited_dirs: HashSet<HgPathBuf>,
+    audited: Mutex<HashSet<HgPathBuf>>,
+    audited_dirs: RwLock<HashSet<HgPathBuf>>,
     root: PathBuf,
 }
 
 impl PathAuditor {
     pub fn new(root: impl AsRef<Path>) -> Self {
         Self {
             root: root.as_ref().to_owned(),
             ..Default::default()
         }
     }
     pub fn audit_path(
-        &mut self,
+        &self,
         path: impl AsRef<HgPath>,
     ) -> Result<(), HgPathError> {
         // TODO windows "localpath" normalization
         let path = path.as_ref();
         if path.is_empty() {
             return Ok(());
         }
         // TODO case normalization
-        if self.audited.contains(path) {
+        if self.audited.lock().unwrap().contains(path) {
             return Ok(());
         }
         // AIX ignores "/" at end of path, others raise EISDIR.
         let last_byte = path.as_bytes()[path.len() - 1];
         if last_byte == b'/' || last_byte == b'\\' {
             return Err(HgPathError::EndsWithSlash(path.to_owned()));
         }
         let parts: Vec<_> = path
             .as_bytes()
             .split(|b| std::path::is_separator(*b as char))
             .collect();
 
         let first_component = lower_clean(parts[0]);
         let first_component = first_component.as_slice();
         if !path.split_drive().0.is_empty()
             || (first_component == b".hg"
                 || first_component == b".hg."
                 || first_component == b"")
             || parts.iter().any(|c| c == b"..")
         {
             return Err(HgPathError::InsideDotHg(path.to_owned()));
         }
 
         // Windows shortname aliases
         for part in parts.iter() {
             if part.contains(&b'~') {
                 let mut split = part.splitn(2, |b| *b == b'~');
                 let first =
                     split.next().unwrap().to_owned().to_ascii_uppercase();
                 let last = split.next().unwrap();
                 if last.iter().all(u8::is_ascii_digit)
                     && (first == b"HG" || first == b"HG8B6C")
                 {
                     return Err(HgPathError::ContainsIllegalComponent(
                         path.to_owned(),
                     ));
                 }
             }
         }
         let lower_path = lower_clean(path.as_bytes());
         if find_slice_in_slice(&lower_path, b".hg").is_some() {
             let lower_parts: Vec<_> = path
                 .as_bytes()
                 .split(|b| std::path::is_separator(*b as char))
                 .collect();
             for pattern in [b".hg".to_vec(), b".hg.".to_vec()].iter() {
                 if let Some(pos) = lower_parts[1..]
                     .iter()
                     .position(|part| part == &pattern.as_slice())
                 {
                     let base = lower_parts[..=pos]
                         .iter()
                         .fold(HgPathBuf::new(), |acc, p| {
                             acc.join(HgPath::new(p))
                         });
                     return Err(HgPathError::IsInsideNestedRepo {
                         path: path.to_owned(),
                         nested_repo: base,
                     });
                 }
             }
         }
 
         let parts = &parts[..parts.len().saturating_sub(1)];
 
         // We don't want to add "foo/bar/baz" to `audited_dirs` before checking
         // if there's a "foo/.hg" directory. This also means we won't
         // accidentally traverse a symlink into some other filesystem (which
         // is potentially expensive to access).
         for index in 0..parts.len() {
             let prefix = &parts[..index + 1].join(&b'/');
             let prefix = HgPath::new(prefix);
-            if self.audited_dirs.contains(prefix) {
+            if self.audited_dirs.read().unwrap().contains(prefix) {
                 continue;
             }
             self.check_filesystem(&prefix, &path)?;
-            self.audited_dirs.insert(prefix.to_owned());
+            self.audited_dirs.write().unwrap().insert(prefix.to_owned());
         }
 
-        self.audited.insert(path.to_owned());
+        self.audited.lock().unwrap().insert(path.to_owned());
 
         Ok(())
     }
 
     pub fn check_filesystem(
         &self,
         prefix: impl AsRef<HgPath>,
         path: impl AsRef<HgPath>,
     ) -> Result<(), HgPathError> {
         let prefix = prefix.as_ref();
         let path = path.as_ref();
         let current_path = self.root.join(
             hg_path_to_path_buf(prefix)
                 .map_err(|_| HgPathError::NotFsCompliant(path.to_owned()))?,
         );
         match std::fs::symlink_metadata(&current_path) {
             Err(e) => {
                 // EINVAL can be raised as invalid path syntax under win32.
                 if e.kind() != std::io::ErrorKind::NotFound
                     && e.kind() != std::io::ErrorKind::InvalidInput
                     && e.raw_os_error() != Some(20)
                 {
                     // Rust does not yet have an `ErrorKind` for
                     // `NotADirectory` (errno 20)
                     // It happens if the dirstate contains `foo/bar` and
                     // foo is not a directory
                     return Err(HgPathError::NotFsCompliant(path.to_owned()));
                 }
             }
             Ok(meta) => {
                 if meta.file_type().is_symlink() {
                     return Err(HgPathError::TraversesSymbolicLink {
                         path: path.to_owned(),
                         symlink: prefix.to_owned(),
                     });
                 }
                 if meta.file_type().is_dir()
                     && current_path.join(".hg").is_dir()
                 {
                     return Err(HgPathError::IsInsideNestedRepo {
                         path: path.to_owned(),
                         nested_repo: prefix.to_owned(),
                     });
                 }
             }
         };
 
         Ok(())
     }
 
-    pub fn check(&mut self, path: impl AsRef<HgPath>) -> bool {
+    pub fn check(&self, path: impl AsRef<HgPath>) -> bool {
         self.audit_path(path).is_ok()
     }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::utils::files::get_path_from_bytes;
     use crate::utils::hg_path::path_to_hg_path_buf;
 
     #[test]
     fn test_path_auditor() {
-        let mut auditor = PathAuditor::new(get_path_from_bytes(b"/tmp"));
+        let auditor = PathAuditor::new(get_path_from_bytes(b"/tmp"));
 
         let path = HgPath::new(b".hg/00changelog.i");
         assert_eq!(
             auditor.audit_path(path),
             Err(HgPathError::InsideDotHg(path.to_owned()))
         );
         let path = HgPath::new(b"this/is/nested/.hg/thing.txt");
         assert_eq!(
             auditor.audit_path(path),
             Err(HgPathError::IsInsideNestedRepo {
                 path: path.to_owned(),
                 nested_repo: HgPathBuf::from_bytes(b"this/is/nested")
             })
         );
 
         use std::fs::{create_dir, File};
         use tempfile::tempdir;
 
         let base_dir = tempdir().unwrap();
         let base_dir_path = base_dir.path();
         let a = base_dir_path.join("a");
         let b = base_dir_path.join("b");
         create_dir(&a).unwrap();
         let in_a_path = a.join("in_a");
         File::create(in_a_path).unwrap();
 
         // TODO make portable
         std::os::unix::fs::symlink(&a, &b).unwrap();
 
         let buf = b.join("in_a").components().skip(2).collect::<PathBuf>();
         eprintln!("buf: {}", buf.display());
         let path = path_to_hg_path_buf(buf).unwrap();
         assert_eq!(
             auditor.audit_path(&path),
             Err(HgPathError::TraversesSymbolicLink {
                 path: path,
                 symlink: path_to_hg_path_buf(
                     b.components().skip(2).collect::<PathBuf>()
                 )
                 .unwrap()
             })
         );
     }
 }