// status.rs // // Copyright 2019 Raphaël Gomès // // This software may be used and distributed according to the terms of the // GNU General Public License version 2 or any later version. //! Rust implementation of dirstate.status (dirstate.py). //! It is currently missing a lot of functionality compared to the Python one //! and will only be triggered in narrow cases. use crate::{ dirstate::SIZE_FROM_OTHER_PARENT, filepatterns::PatternFileWarning, matchers::{get_ignore_function, Matcher, VisitChildrenSet}, utils::{ files::{find_dirs, HgMetadata}, hg_path::{ hg_path_to_path_buf, os_string_to_hg_path_buf, HgPath, HgPathBuf, HgPathError, }, path_auditor::PathAuditor, }, CopyMap, DirstateEntry, DirstateMap, EntryState, FastHashMap, PatternError, }; use lazy_static::lazy_static; use micro_timer::timed; use rayon::prelude::*; use std::{ borrow::Cow, collections::HashSet, fs::{read_dir, DirEntry}, io::ErrorKind, ops::Deref, path::{Path, PathBuf}, }; /// Wrong type of file from a `BadMatch` /// Note: a lot of those don't exist on all platforms. #[derive(Debug, Copy, Clone)] pub enum BadType { CharacterDevice, BlockDevice, FIFO, Socket, Directory, Unknown, } impl ToString for BadType { fn to_string(&self) -> String { match self { BadType::CharacterDevice => "character device", BadType::BlockDevice => "block device", BadType::FIFO => "fifo", BadType::Socket => "socket", BadType::Directory => "directory", BadType::Unknown => "unknown", } .to_string() } } /// Was explicitly matched but cannot be found/accessed #[derive(Debug, Copy, Clone)] pub enum BadMatch { OsError(i32), BadType(BadType), } /// Marker enum used to dispatch new status entries into the right collections. /// Is similar to `crate::EntryState`, but represents the transient state of /// entries during the lifetime of a command. #[derive(Debug, Copy, Clone)] pub enum Dispatch { Unsure, Modified, Added, Removed, Deleted, Clean, Unknown, Ignored, /// Empty dispatch, the file is not worth listing None, /// Was explicitly matched but cannot be found/accessed Bad(BadMatch), Directory { /// True if the directory used to be a file in the dmap so we can say /// that it's been removed. was_file: bool, }, } type IoResult = std::io::Result; /// `Box` is syntactic sugar for `Box`, so add /// an explicit lifetime here to not fight `'static` bounds "out of nowhere". type IgnoreFnType<'a> = Box Fn(&'r HgPath) -> bool + Sync + 'a>; /// Dates and times that are outside the 31-bit signed range are compared /// modulo 2^31. This should prevent hg from behaving badly with very large /// files or corrupt dates while still having a high probability of detecting /// changes. (issue2608) /// TODO I haven't found a way of having `b` be `Into`, since `From` /// is not defined for `i32`, and there is no `As` trait. This forces the /// caller to cast `b` as `i32`. fn mod_compare(a: i32, b: i32) -> bool { a & i32::max_value() != b & i32::max_value() } /// Return a sorted list containing information about the entries /// in the directory. /// /// * `skip_dot_hg` - Return an empty vec if `path` contains a `.hg` directory fn list_directory( path: impl AsRef, skip_dot_hg: bool, ) -> std::io::Result> { let mut results = vec![]; let entries = read_dir(path.as_ref())?; for entry in entries { let entry = entry?; let filename = os_string_to_hg_path_buf(entry.file_name())?; let file_type = entry.file_type()?; if skip_dot_hg && filename.as_bytes() == b".hg" && file_type.is_dir() { return Ok(vec![]); } else { results.push((filename, entry)) } } results.sort_unstable_by_key(|e| e.0.clone()); Ok(results) } /// The file corresponding to the dirstate entry was found on the filesystem. fn dispatch_found( filename: impl AsRef, entry: DirstateEntry, metadata: HgMetadata, copy_map: &CopyMap, options: StatusOptions, ) -> Dispatch { let DirstateEntry { state, mode, mtime, size, } = entry; let HgMetadata { st_mode, st_size, st_mtime, .. } = metadata; match state { EntryState::Normal => { let size_changed = mod_compare(size, st_size as i32); let mode_changed = (mode ^ st_mode as i32) & 0o100 != 0o000 && options.check_exec; let metadata_changed = size >= 0 && (size_changed || mode_changed); let other_parent = size == SIZE_FROM_OTHER_PARENT; if metadata_changed || other_parent || copy_map.contains_key(filename.as_ref()) { Dispatch::Modified } else if mod_compare(mtime, st_mtime as i32) || st_mtime == options.last_normal_time { // the file may have just been marked as normal and // it may have changed in the same second without // changing its size. This can happen if we quickly // do multiple commits. Force lookup, so we don't // miss such a racy file change. Dispatch::Unsure } else if options.list_clean { Dispatch::Clean } else { Dispatch::None } } EntryState::Merged => Dispatch::Modified, EntryState::Added => Dispatch::Added, EntryState::Removed => Dispatch::Removed, EntryState::Unknown => Dispatch::Unknown, } } /// The file corresponding to this Dirstate entry is missing. fn dispatch_missing(state: EntryState) -> Dispatch { match state { // File was removed from the filesystem during commands EntryState::Normal | EntryState::Merged | EntryState::Added => { Dispatch::Deleted } // File was removed, everything is normal EntryState::Removed => Dispatch::Removed, // File is unknown to Mercurial, everything is normal EntryState::Unknown => Dispatch::Unknown, } } lazy_static! { static ref DEFAULT_WORK: HashSet<&'static HgPath> = { let mut h = HashSet::new(); h.insert(HgPath::new(b"")); h }; } #[derive(Debug, Copy, Clone)] pub struct StatusOptions { /// Remember the most recent modification timeslot for status, to make /// sure we won't miss future size-preserving file content modifications /// that happen within the same timeslot. pub last_normal_time: i64, /// Whether we are on a filesystem with UNIX-like exec flags pub check_exec: bool, pub list_clean: bool, pub list_unknown: bool, pub list_ignored: bool, /// Whether to collect traversed dirs for applying a callback later. /// Used by `hg purge` for example. pub collect_traversed_dirs: bool, } #[derive(Debug)] pub struct DirstateStatus<'a> { pub modified: Vec>, pub added: Vec>, pub removed: Vec>, pub deleted: Vec>, pub clean: Vec>, pub ignored: Vec>, pub unknown: Vec>, pub bad: Vec<(Cow<'a, HgPath>, BadMatch)>, /// Only filled if `collect_traversed_dirs` is `true` pub traversed: Vec, } #[derive(Debug)] pub enum StatusError { IO(std::io::Error), Path(HgPathError), Pattern(PatternError), } pub type StatusResult = Result; impl From for StatusError { fn from(e: PatternError) -> Self { StatusError::Pattern(e) } } impl From for StatusError { fn from(e: HgPathError) -> Self { StatusError::Path(e) } } impl From for StatusError { fn from(e: std::io::Error) -> Self { StatusError::IO(e) } } impl ToString for StatusError { fn to_string(&self) -> String { match self { StatusError::IO(e) => e.to_string(), StatusError::Path(e) => e.to_string(), StatusError::Pattern(e) => e.to_string(), } } } pub struct Status<'a, M: Matcher + Sync> { dmap: &'a DirstateMap, matcher: &'a M, root_dir: PathBuf, options: StatusOptions, ignore_fn: IgnoreFnType<'a>, } impl<'a, M> Status<'a, M> where M: Matcher + Sync, { pub fn new( dmap: &'a DirstateMap, matcher: &'a M, root_dir: PathBuf, ignore_files: Vec, options: StatusOptions, ) -> StatusResult<(Self, Vec)> { // Needs to outlive `dir_ignore_fn` since it's captured. let (ignore_fn, warnings): (IgnoreFnType, _) = if options.list_ignored || options.list_unknown { get_ignore_function(ignore_files, &root_dir)? } else { (Box::new(|&_| true), vec![]) }; Ok(( Self { dmap, matcher, root_dir, options, ignore_fn, }, warnings, )) } pub fn is_ignored(&self, path: impl AsRef) -> bool { (self.ignore_fn)(path.as_ref()) } /// Is the path or one of its ancestors ignored? pub fn dir_ignore(&self, dir: impl AsRef) -> bool { // Only involve ignore mechanism if we're listing unknowns or ignored. if self.options.list_ignored || self.options.list_unknown { if self.is_ignored(&dir) { true } else { for p in find_dirs(dir.as_ref()) { if self.is_ignored(p) { return true; } } false } } else { true } } /// Get stat data about the files explicitly specified by match. /// TODO subrepos #[timed] pub fn walk_explicit( &self, traversed_sender: crossbeam::Sender, ) -> ( Vec<(Cow<'a, HgPath>, Dispatch)>, Vec<(Cow<'a, HgPath>, Dispatch)>, ) { self.matcher .file_set() .unwrap_or(&DEFAULT_WORK) .par_iter() .map(|&filename| -> Option> { // TODO normalization let normalized = filename; let buf = match hg_path_to_path_buf(normalized) { Ok(x) => x, Err(e) => return Some(Err(e.into())), }; let target = self.root_dir.join(buf); let st = target.symlink_metadata(); let in_dmap = self.dmap.get(normalized); match st { Ok(meta) => { let file_type = meta.file_type(); return if file_type.is_file() || file_type.is_symlink() { if let Some(entry) = in_dmap { return Some(Ok(( Cow::Borrowed(normalized), dispatch_found( &normalized, *entry, HgMetadata::from_metadata(meta), &self.dmap.copy_map, self.options, ), ))); } Some(Ok(( Cow::Borrowed(normalized), Dispatch::Unknown, ))) } else if file_type.is_dir() { if self.options.collect_traversed_dirs { traversed_sender .send(normalized.to_owned()) .expect("receiver should outlive sender"); } Some(Ok(( Cow::Borrowed(normalized), Dispatch::Directory { was_file: in_dmap.is_some(), }, ))) } else { Some(Ok(( Cow::Borrowed(normalized), Dispatch::Bad(BadMatch::BadType( // TODO do more than unknown // Support for all `BadType` variant // varies greatly between platforms. // So far, no tests check the type and // this should be good enough for most // users. BadType::Unknown, )), ))) }; } Err(_) => { if let Some(entry) = in_dmap { return Some(Ok(( Cow::Borrowed(normalized), dispatch_missing(entry.state), ))); } } }; None }) .flatten() .filter_map(Result::ok) .partition(|(_, dispatch)| match dispatch { Dispatch::Directory { .. } => true, _ => false, }) } /// Walk the working directory recursively to look for changes compared to /// the current `DirstateMap`. /// /// This takes a mutable reference to the results to account for the /// `extend` in timings #[timed] pub fn traverse( &self, path: impl AsRef, old_results: &FastHashMap, Dispatch>, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>, traversed_sender: crossbeam::Sender, ) -> IoResult<()> { // The traversal is done in parallel, so use a channel to gather // entries. `crossbeam::Sender` is `Sync`, while `mpsc::Sender` // is not. let (files_transmitter, files_receiver) = crossbeam::channel::unbounded(); self.traverse_dir( &files_transmitter, path, &old_results, traversed_sender, )?; // Disconnect the channel so the receiver stops waiting drop(files_transmitter); // TODO don't collect. Find a way of replicating the behavior of // `itertools::process_results`, but for `rayon::ParallelIterator` let new_results: IoResult, Dispatch)>> = files_receiver .into_iter() .map(|item| { let (f, d) = item?; Ok((Cow::Owned(f), d)) }) .collect(); results.par_extend(new_results?); Ok(()) } /// Dispatch a single entry (file, folder, symlink...) found during /// `traverse`. If the entry is a folder that needs to be traversed, it /// will be handled in a separate thread. fn handle_traversed_entry<'b>( &'a self, scope: &rayon::Scope<'b>, files_sender: &'b crossbeam::Sender>, old_results: &'a FastHashMap, Dispatch>, filename: HgPathBuf, dir_entry: DirEntry, traversed_sender: crossbeam::Sender, ) -> IoResult<()> where 'a: 'b, { let file_type = dir_entry.file_type()?; let entry_option = self.dmap.get(&filename); if filename.as_bytes() == b".hg" { // Could be a directory or a symlink return Ok(()); } if file_type.is_dir() { self.handle_traversed_dir( scope, files_sender, old_results, entry_option, filename, traversed_sender, ); } else if file_type.is_file() || file_type.is_symlink() { if let Some(entry) = entry_option { if self.matcher.matches_everything() || self.matcher.matches(&filename) { let metadata = dir_entry.metadata()?; files_sender .send(Ok(( filename.to_owned(), dispatch_found( &filename, *entry, HgMetadata::from_metadata(metadata), &self.dmap.copy_map, self.options, ), ))) .unwrap(); } } else if (self.matcher.matches_everything() || self.matcher.matches(&filename)) && !self.is_ignored(&filename) { if (self.options.list_ignored || self.matcher.exact_match(&filename)) && self.dir_ignore(&filename) { if self.options.list_ignored { files_sender .send(Ok((filename.to_owned(), Dispatch::Ignored))) .unwrap(); } } else if self.options.list_unknown { files_sender .send(Ok((filename.to_owned(), Dispatch::Unknown))) .unwrap(); } } else if self.is_ignored(&filename) && self.options.list_ignored { files_sender .send(Ok((filename.to_owned(), Dispatch::Ignored))) .unwrap(); } } else if let Some(entry) = entry_option { // Used to be a file or a folder, now something else. if self.matcher.matches_everything() || self.matcher.matches(&filename) { files_sender .send(Ok(( filename.to_owned(), dispatch_missing(entry.state), ))) .unwrap(); } } Ok(()) } /// A directory was found in the filesystem and needs to be traversed fn handle_traversed_dir<'b>( &'a self, scope: &rayon::Scope<'b>, files_sender: &'b crossbeam::Sender>, old_results: &'a FastHashMap, Dispatch>, entry_option: Option<&'a DirstateEntry>, directory: HgPathBuf, traversed_sender: crossbeam::Sender, ) where 'a: 'b, { scope.spawn(move |_| { // Nested `if` until `rust-lang/rust#53668` is stable if let Some(entry) = entry_option { // Used to be a file, is now a folder if self.matcher.matches_everything() || self.matcher.matches(&directory) { files_sender .send(Ok(( directory.to_owned(), dispatch_missing(entry.state), ))) .unwrap(); } } // Do we need to traverse it? if !self.is_ignored(&directory) || self.options.list_ignored { self.traverse_dir( files_sender, directory, &old_results, traversed_sender, ) .unwrap_or_else(|e| files_sender.send(Err(e)).unwrap()) } }); } /// Decides whether the directory needs to be listed, and if so handles the /// entries in a separate thread. fn traverse_dir( &self, files_sender: &crossbeam::Sender>, directory: impl AsRef, old_results: &FastHashMap, Dispatch>, traversed_sender: crossbeam::Sender, ) -> IoResult<()> { let directory = directory.as_ref(); if self.options.collect_traversed_dirs { traversed_sender .send(directory.to_owned()) .expect("receiver should outlive sender"); } let visit_entries = match self.matcher.visit_children_set(directory) { VisitChildrenSet::Empty => return Ok(()), VisitChildrenSet::This | VisitChildrenSet::Recursive => None, VisitChildrenSet::Set(set) => Some(set), }; let buf = hg_path_to_path_buf(directory)?; let dir_path = self.root_dir.join(buf); let skip_dot_hg = !directory.as_bytes().is_empty(); let entries = match list_directory(dir_path, skip_dot_hg) { Err(e) => match e.kind() { ErrorKind::NotFound | ErrorKind::PermissionDenied => { files_sender .send(Ok(( directory.to_owned(), Dispatch::Bad(BadMatch::OsError( // Unwrapping here is OK because the error // always is a // real os error e.raw_os_error().unwrap(), )), ))) .unwrap(); return Ok(()); } _ => return Err(e), }, Ok(entries) => entries, }; rayon::scope(|scope| -> IoResult<()> { for (filename, dir_entry) in entries { if let Some(ref set) = visit_entries { if !set.contains(filename.deref()) { continue; } } // TODO normalize let filename = if directory.is_empty() { filename.to_owned() } else { directory.join(&filename) }; if !old_results.contains_key(filename.deref()) { self.handle_traversed_entry( scope, files_sender, old_results, filename, dir_entry, traversed_sender.clone(), )?; } } Ok(()) }) } /// This takes a mutable reference to the results to account for the /// `extend` in timings #[timed] fn handle_unknowns( &self, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>, ) -> IoResult<()> { let to_visit: Vec<(&HgPath, &DirstateEntry)> = if results.is_empty() && self.matcher.matches_everything() { self.dmap.iter().map(|(f, e)| (f.deref(), e)).collect() } else { // Only convert to a hashmap if needed. let old_results: FastHashMap<_, _> = results.iter().cloned().collect(); self.dmap .iter() .filter_map(move |(f, e)| { if !old_results.contains_key(f.deref()) && self.matcher.matches(f) { Some((f.deref(), e)) } else { None } }) .collect() }; // We walked all dirs under the roots that weren't ignored, and // everything that matched was stat'ed and is already in results. // The rest must thus be ignored or under a symlink. let path_auditor = PathAuditor::new(&self.root_dir); // TODO don't collect. Find a way of replicating the behavior of // `itertools::process_results`, but for `rayon::ParallelIterator` let new_results: IoResult> = to_visit .into_par_iter() .filter_map(|(filename, entry)| -> Option> { // Report ignored items in the dmap as long as they are not // under a symlink directory. if path_auditor.check(filename) { // TODO normalize for case-insensitive filesystems let buf = match hg_path_to_path_buf(filename) { Ok(x) => x, Err(e) => return Some(Err(e.into())), }; Some(Ok(( Cow::Borrowed(filename), match self.root_dir.join(&buf).symlink_metadata() { // File was just ignored, no links, and exists Ok(meta) => { let metadata = HgMetadata::from_metadata(meta); dispatch_found( filename, *entry, metadata, &self.dmap.copy_map, self.options, ) } // File doesn't exist Err(_) => dispatch_missing(entry.state), }, ))) } else { // It's either missing or under a symlink directory which // we, in this case, report as missing. Some(Ok(( Cow::Borrowed(filename), dispatch_missing(entry.state), ))) } }) .collect(); results.par_extend(new_results?); Ok(()) } /// This takes a mutable reference to the results to account for the /// `extend` in timings #[timed] fn extend_from_dmap( &self, results: &mut Vec<(Cow<'a, HgPath>, Dispatch)>, ) { results.par_extend(self.dmap.par_iter().flat_map( move |(filename, entry)| { let filename: &HgPath = filename; let filename_as_path = hg_path_to_path_buf(filename)?; let meta = self.root_dir.join(filename_as_path).symlink_metadata(); match meta { Ok(ref m) if !(m.file_type().is_file() || m.file_type().is_symlink()) => { Ok(( Cow::Borrowed(filename), dispatch_missing(entry.state), )) } Ok(m) => Ok(( Cow::Borrowed(filename), dispatch_found( filename, *entry, HgMetadata::from_metadata(m), &self.dmap.copy_map, self.options, ), )), Err(ref e) if e.kind() == ErrorKind::NotFound || e.raw_os_error() == Some(20) => { // Rust does not yet have an `ErrorKind` for // `NotADirectory` (errno 20) // It happens if the dirstate contains `foo/bar` // and foo is not a // directory Ok(( Cow::Borrowed(filename), dispatch_missing(entry.state), )) } Err(e) => Err(e), } }, )); } } #[timed] fn build_response<'a>( results: impl IntoIterator, Dispatch)>, traversed: Vec, ) -> (Vec>, DirstateStatus<'a>) { let mut lookup = vec![]; let mut modified = vec![]; let mut added = vec![]; let mut removed = vec![]; let mut deleted = vec![]; let mut clean = vec![]; let mut ignored = vec![]; let mut unknown = vec![]; let mut bad = vec![]; for (filename, dispatch) in results.into_iter() { match dispatch { Dispatch::Unknown => unknown.push(filename), Dispatch::Unsure => lookup.push(filename), Dispatch::Modified => modified.push(filename), Dispatch::Added => added.push(filename), Dispatch::Removed => removed.push(filename), Dispatch::Deleted => deleted.push(filename), Dispatch::Clean => clean.push(filename), Dispatch::Ignored => ignored.push(filename), Dispatch::None => {} Dispatch::Bad(reason) => bad.push((filename, reason)), Dispatch::Directory { .. } => {} } } ( lookup, DirstateStatus { modified, added, removed, deleted, clean, ignored, unknown, bad, traversed, }, ) } /// Get the status of files in the working directory. /// /// This is the current entry-point for `hg-core` and is realistically unusable /// outside of a Python context because its arguments need to provide a lot of /// information that will not be necessary in the future. #[timed] pub fn status<'a>( dmap: &'a DirstateMap, matcher: &'a (impl Matcher + Sync), root_dir: PathBuf, ignore_files: Vec, options: StatusOptions, ) -> StatusResult<( (Vec>, DirstateStatus<'a>), Vec, )> { let (traversed_sender, traversed_receiver) = crossbeam::channel::unbounded(); let (st, warnings) = Status::new(dmap, matcher, root_dir, ignore_files, options)?; // Step 1: check the files explicitly mentioned by the user let (work, mut results) = st.walk_explicit(traversed_sender.clone()); if !work.is_empty() { // Hashmaps are quite a bit slower to build than vecs, so only build it // if needed. let old_results = results.iter().cloned().collect(); // Step 2: recursively check the working directory for changes if // needed for (dir, dispatch) in work { match dispatch { Dispatch::Directory { was_file } => { if was_file { results.push((dir.to_owned(), Dispatch::Removed)); } if options.list_ignored || options.list_unknown && !st.dir_ignore(&dir) { st.traverse( &dir, &old_results, &mut results, traversed_sender.clone(), )?; } } _ => unreachable!("There can only be directories in `work`"), } } } if !matcher.is_exact() { // Step 3: Check the remaining files from the dmap. // If a dmap file is not in results yet, it was either // a) not matched b) ignored, c) missing, or d) under a // symlink directory. if options.list_unknown { st.handle_unknowns(&mut results)?; } else { // We may not have walked the full directory tree above, so stat // and check everything we missed. st.extend_from_dmap(&mut results); } } drop(traversed_sender); let traversed = traversed_receiver.into_iter().collect(); Ok((build_response(results, traversed), warnings)) }