rust: simplify pattern file parsing

Previously we kept the pattern syntax as a &[u8] until the last
possible moment, which meant we had to handle potential errors when
parsing that &[u8].  But such errors could never actually occur, given
the structure of the code.

Now we parse it eagerly (in two places) and pass it around as a
PatternSyntax, so we can delete some error handling code.

parse_one_pattern is in particular useful for parsing patterns passed
on the command line, as we'll support later in this series.

rust/hg-core/src/filepatterns.rs

@@ -1,776 +1,807 b''
 // filepatterns.rs
 //
 // Copyright 2019 Raphaël Gomès <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Handling of Mercurial-specific patterns.
 
 use crate::{
     utils::{
         files::{canonical_path, get_bytes_from_path, get_path_from_bytes},
         hg_path::{path_to_hg_path_buf, HgPathBuf, HgPathError},
         SliceExt,
     },
     FastHashMap, PatternError,
 };
 use lazy_static::lazy_static;
 use regex::bytes::{NoExpand, Regex};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::vec::Vec;
 
 lazy_static! {
     static ref RE_ESCAPE: Vec<Vec<u8>> = {
         let mut v: Vec<Vec<u8>> = (0..=255).map(|byte| vec![byte]).collect();
         let to_escape = b"()[]{}?*+-|^$\\.&~# \t\n\r\x0b\x0c";
         for byte in to_escape {
             v[*byte as usize].insert(0, b'\\');
         }
         v
     };
 }
 
 /// These are matched in order
 const GLOB_REPLACEMENTS: &[(&[u8], &[u8])] =
     &[(b"*/", b"(?:.*/)?"), (b"*", b".*"), (b"", b"[^/]*")];
 
 /// Appended to the regexp of globs
 const GLOB_SUFFIX: &[u8; 7] = b"(?:/|$)";
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum PatternSyntax {
     /// A regular expression
     Regexp,
     /// Glob that matches at the front of the path
     RootGlob,
     /// Glob that matches at any suffix of the path (still anchored at
     /// slashes)
     Glob,
     /// a path relative to repository root, which is matched recursively
     Path,
     /// a single exact path relative to repository root
     FilePath,
     /// A path relative to cwd
     RelPath,
     /// an unrooted glob (*.rs matches Rust files in all dirs)
     RelGlob,
     /// A regexp that needn't match the start of a name
     RelRegexp,
     /// A path relative to repository root, which is matched non-recursively
     /// (will not match subdirectories)
     RootFiles,
     /// A file of patterns to read and include
     Include,
     /// A file of patterns to match against files under the same directory
     SubInclude,
     /// SubInclude with the result of parsing the included file
     ///
     /// Note: there is no ExpandedInclude because that expansion can be done
     /// in place by replacing the Include pattern by the included patterns.
     /// SubInclude requires more handling.
     ///
     /// Note: `Box` is used to minimize size impact on other enum variants
     ExpandedSubInclude(Box<SubInclude>),
 }
 
 /// Transforms a glob pattern into a regex
 fn glob_to_re(pat: &[u8]) -> Vec<u8> {
     let mut input = pat;
     let mut res: Vec<u8> = vec![];
     let mut group_depth = 0;
 
     while let Some((c, rest)) = input.split_first() {
         input = rest;
 
         match c {
             b'*' => {
                 for (source, repl) in GLOB_REPLACEMENTS {
                     if let Some(rest) = input.drop_prefix(source) {
                         input = rest;
                         res.extend(*repl);
                         break;
                     }
                 }
             }
             b'?' => res.extend(b"."),
             b'[' => {
                 match input.iter().skip(1).position(|b| *b == b']') {
                     None => res.extend(b"\\["),
                     Some(end) => {
                         // Account for the one we skipped
                         let end = end + 1;
 
                         res.extend(b"[");
 
                         for (i, b) in input[..end].iter().enumerate() {
                             if *b == b'!' && i == 0 {
                                 res.extend(b"^")
                             } else if *b == b'^' && i == 0 {
                                 res.extend(b"\\^")
                             } else if *b == b'\\' {
                                 res.extend(b"\\\\")
                             } else {
                                 res.push(*b)
                             }
                         }
                         res.extend(b"]");
                         input = &input[end + 1..];
                     }
                 }
             }
             b'{' => {
                 group_depth += 1;
                 res.extend(b"(?:")
             }
             b'}' if group_depth > 0 => {
                 group_depth -= 1;
                 res.extend(b")");
             }
             b',' if group_depth > 0 => res.extend(b"|"),
             b'\\' => {
                 let c = {
                     if let Some((c, rest)) = input.split_first() {
                         input = rest;
                         c
                     } else {
                         c
                     }
                 };
                 res.extend(&RE_ESCAPE[*c as usize])
             }
             _ => res.extend(&RE_ESCAPE[*c as usize]),
         }
     }
     res
 }
 
 fn escape_pattern(pattern: &[u8]) -> Vec<u8> {
     pattern
         .iter()
         .flat_map(|c| RE_ESCAPE[*c as usize].clone())
         .collect()
 }
 
 pub fn parse_pattern_syntax(
     kind: &[u8],
 ) -> Result<PatternSyntax, PatternError> {
     match kind {
         b"re:" => Ok(PatternSyntax::Regexp),
         b"path:" => Ok(PatternSyntax::Path),
         b"filepath:" => Ok(PatternSyntax::FilePath),
         b"relpath:" => Ok(PatternSyntax::RelPath),
         b"rootfilesin:" => Ok(PatternSyntax::RootFiles),
         b"relglob:" => Ok(PatternSyntax::RelGlob),
         b"relre:" => Ok(PatternSyntax::RelRegexp),
         b"glob:" => Ok(PatternSyntax::Glob),
         b"rootglob:" => Ok(PatternSyntax::RootGlob),
         b"include:" => Ok(PatternSyntax::Include),
         b"subinclude:" => Ok(PatternSyntax::SubInclude),
         _ => Err(PatternError::UnsupportedSyntax(
             String::from_utf8_lossy(kind).to_string(),
         )),
     }
 }
 
 lazy_static! {
     static ref FLAG_RE: Regex = Regex::new(r"^\(\?[aiLmsux]+\)").unwrap();
 }
 
 /// Builds the regex that corresponds to the given pattern.
 /// If within a `syntax: regexp` context, returns the pattern,
 /// otherwise, returns the corresponding regex.
 fn _build_single_regex(entry: &IgnorePattern) -> Vec<u8> {
     let IgnorePattern {
         syntax, pattern, ..
     } = entry;
     if pattern.is_empty() {
         return vec![];
     }
     match syntax {
         PatternSyntax::Regexp => pattern.to_owned(),
         PatternSyntax::RelRegexp => {
             // The `regex` crate accepts `**` while `re2` and Python's `re`
             // do not. Checking for `*` correctly triggers the same error all
             // engines.
             if pattern[0] == b'^'
                 || pattern[0] == b'*'
                 || pattern.starts_with(b".*")
             {
                 return pattern.to_owned();
             }
             match FLAG_RE.find(pattern) {
                 Some(mat) => {
                     let s = mat.start();
                     let e = mat.end();
                     [
                         &b"(?"[..],
                         &pattern[s + 2..e - 1],
                         &b":"[..],
                         if pattern[e] == b'^'
                             || pattern[e] == b'*'
                             || pattern[e..].starts_with(b".*")
                         {
                             &b""[..]
                         } else {
                             &b".*"[..]
                         },
                         &pattern[e..],
                         &b")"[..],
                     ]
                     .concat()
                 }
                 None => [&b".*"[..], pattern].concat(),
             }
         }
         PatternSyntax::Path | PatternSyntax::RelPath => {
             if pattern == b"." {
                 return vec![];
             }
             [escape_pattern(pattern).as_slice(), b"(?:/|$)"].concat()
         }
         PatternSyntax::RootFiles => {
             let mut res = if pattern == b"." {
                 vec![]
             } else {
                 // Pattern is a directory name.
                 [escape_pattern(pattern).as_slice(), b"/"].concat()
             };
 
             // Anything after the pattern must be a non-directory.
             res.extend(b"[^/]+$");
             res
         }
         PatternSyntax::RelGlob => {
             let glob_re = glob_to_re(pattern);
             if let Some(rest) = glob_re.drop_prefix(b"[^/]*") {
                 [b".*", rest, GLOB_SUFFIX].concat()
             } else {
                 [b"(?:.*/)?", glob_re.as_slice(), GLOB_SUFFIX].concat()
             }
         }
         PatternSyntax::Glob | PatternSyntax::RootGlob => {
             [glob_to_re(pattern).as_slice(), GLOB_SUFFIX].concat()
         }
         PatternSyntax::Include
         | PatternSyntax::SubInclude
         | PatternSyntax::ExpandedSubInclude(_)
         | PatternSyntax::FilePath => unreachable!(),
     }
 }
 
 const GLOB_SPECIAL_CHARACTERS: [u8; 7] =
     [b'*', b'?', b'[', b']', b'{', b'}', b'\\'];
 
 /// TODO support other platforms
 #[cfg(unix)]
 pub fn normalize_path_bytes(bytes: &[u8]) -> Vec<u8> {
     if bytes.is_empty() {
         return b".".to_vec();
     }
     let sep = b'/';
 
     let mut initial_slashes = bytes.iter().take_while(|b| **b == sep).count();
     if initial_slashes > 2 {
         // POSIX allows one or two initial slashes, but treats three or more
         // as single slash.
         initial_slashes = 1;
     }
     let components = bytes
         .split(|b| *b == sep)
         .filter(|c| !(c.is_empty() || c == b"."))
         .fold(vec![], |mut acc, component| {
             if component != b".."
                 || (initial_slashes == 0 && acc.is_empty())
                 || (!acc.is_empty() && acc[acc.len() - 1] == b"..")
             {
                 acc.push(component)
             } else if !acc.is_empty() {
                 acc.pop();
             }
             acc
         });
     let mut new_bytes = components.join(&sep);
 
     if initial_slashes > 0 {
         let mut buf: Vec<_> = (0..initial_slashes).map(|_| sep).collect();
         buf.extend(new_bytes);
         new_bytes = buf;
     }
     if new_bytes.is_empty() {
         b".".to_vec()
     } else {
         new_bytes
     }
 }
 
 /// Wrapper function to `_build_single_regex` that short-circuits 'exact' globs
 /// that don't need to be transformed into a regex.
 pub fn build_single_regex(
     entry: &IgnorePattern,
 ) -> Result<Option<Vec<u8>>, PatternError> {
     let IgnorePattern {
         pattern, syntax, ..
     } = entry;
     let pattern = match syntax {
         PatternSyntax::RootGlob
         | PatternSyntax::Path
         | PatternSyntax::RelGlob
         | PatternSyntax::RootFiles => normalize_path_bytes(pattern),
         PatternSyntax::Include | PatternSyntax::SubInclude => {
             return Err(PatternError::NonRegexPattern(entry.clone()))
         }
         _ => pattern.to_owned(),
     };
     let is_simple_rootglob = *syntax == PatternSyntax::RootGlob
         && !pattern.iter().any(|b| GLOB_SPECIAL_CHARACTERS.contains(b));
     if is_simple_rootglob || syntax == &PatternSyntax::FilePath {
         Ok(None)
     } else {
         let mut entry = entry.clone();
         entry.pattern = pattern;
         Ok(Some(_build_single_regex(&entry)))
     }
 }
 
 lazy_static! {
-    static ref SYNTAXES: FastHashMap<&'static [u8], &'static [u8]> = {
+    static ref SYNTAXES: FastHashMap<&'static [u8], PatternSyntax> = {
         let mut m = FastHashMap::default();
 
-        m.insert(b"re".as_ref(), b"relre:".as_ref());
-        m.insert(b"regexp".as_ref(), b"relre:".as_ref());
-        m.insert(b"glob".as_ref(), b"relglob:".as_ref());
-        m.insert(b"rootglob".as_ref(), b"rootglob:".as_ref());
-        m.insert(b"include".as_ref(), b"include:".as_ref());
-        m.insert(b"subinclude".as_ref(), b"subinclude:".as_ref());
-        m.insert(b"path".as_ref(), b"path:".as_ref());
-        m.insert(b"rootfilesin".as_ref(), b"rootfilesin:".as_ref());
+        m.insert(b"re:".as_ref(), PatternSyntax::Regexp);
+        m.insert(b"regexp:".as_ref(), PatternSyntax::Regexp);
+        m.insert(b"path:".as_ref(), PatternSyntax::Path);
+        m.insert(b"filepath:".as_ref(), PatternSyntax::FilePath);
+        m.insert(b"relpath:".as_ref(), PatternSyntax::RelPath);
+        m.insert(b"rootfilesin:".as_ref(), PatternSyntax::RootFiles);
+        m.insert(b"relglob:".as_ref(), PatternSyntax::RelGlob);
+        m.insert(b"relre:".as_ref(), PatternSyntax::RelRegexp);
+        m.insert(b"glob:".as_ref(), PatternSyntax::Glob);
+        m.insert(b"rootglob:".as_ref(), PatternSyntax::RootGlob);
+        m.insert(b"include:".as_ref(), PatternSyntax::Include);
+        m.insert(b"subinclude:".as_ref(), PatternSyntax::SubInclude);
+
         m
     };
 }
 
 #[derive(Debug)]
 pub enum PatternFileWarning {
     /// (file path, syntax bytes)
     InvalidSyntax(PathBuf, Vec<u8>),
     /// File path
     NoSuchFile(PathBuf),
 }
 
+pub fn parse_one_pattern(
+    pattern: &[u8],
+    source: &Path,
+    default: PatternSyntax,
+) -> IgnorePattern {
+    let mut pattern_bytes: &[u8] = pattern;
+    let mut syntax = default;
+
+    for (s, val) in SYNTAXES.iter() {
+        if let Some(rest) = pattern_bytes.drop_prefix(s) {
+            syntax = val.clone();
+            pattern_bytes = rest;
+            break;
+        }
+    }
+
+    let pattern = pattern_bytes.to_vec();
+
+    IgnorePattern {
+        syntax,
+        pattern,
+        source: source.to_owned(),
+    }
+}
+
 pub fn parse_pattern_file_contents(
     lines: &[u8],
     file_path: &Path,
-    default_syntax_override: Option<&[u8]>,
+    default_syntax_override: Option<PatternSyntax>,
     warn: bool,
+    relativize: bool,
 ) -> Result<(Vec<IgnorePattern>, Vec<PatternFileWarning>), PatternError> {
     let comment_regex = Regex::new(r"((?:^|[^\\])(?:\\\\)*)#.*").unwrap();
 
     #[allow(clippy::trivial_regex)]
     let comment_escape_regex = Regex::new(r"\\#").unwrap();
     let mut inputs: Vec<IgnorePattern> = vec![];
     let mut warnings: Vec<PatternFileWarning> = vec![];
 
     let mut current_syntax =
-        default_syntax_override.unwrap_or_else(|| b"relre:".as_ref());
+        default_syntax_override.unwrap_or(PatternSyntax::RelRegexp);
 
-    for (line_number, mut line) in lines.split(|c| *c == b'\n').enumerate() {
-        let line_number = line_number + 1;
-
+    for mut line in lines.split(|c| *c == b'\n') {
         let line_buf;
         if line.contains(&b'#') {
             if let Some(cap) = comment_regex.captures(line) {
                 line = &line[..cap.get(1).unwrap().end()]
             }
             line_buf = comment_escape_regex.replace_all(line, NoExpand(b"#"));
             line = &line_buf;
         }
 
-        let mut line = line.trim_end();
+        let line = line.trim_end();
 
         if line.is_empty() {
             continue;
         }
 
         if let Some(syntax) = line.drop_prefix(b"syntax:") {
             let syntax = syntax.trim();
 
-            if let Some(rel_syntax) = SYNTAXES.get(syntax) {
-                current_syntax = rel_syntax;
+            if let Some(parsed) =
+                SYNTAXES.get([syntax, &b":"[..]].concat().as_slice())
+            {
+                current_syntax = parsed.clone();
             } else if warn {
                 warnings.push(PatternFileWarning::InvalidSyntax(
                     file_path.to_owned(),
                     syntax.to_owned(),
                 ));
             }
-            continue;
+        } else {
+            let pattern = parse_one_pattern(
+                line,
+                file_path,
+                current_syntax.clone(),
+            );
+            inputs.push(if relativize {
+                pattern.to_relative()
+            } else {
+                pattern
+            })
         }
-
-        let mut line_syntax: &[u8] = current_syntax;
-
-        for (s, rels) in SYNTAXES.iter() {
-            if let Some(rest) = line.drop_prefix(rels) {
-                line_syntax = rels;
-                line = rest;
-                break;
-            }
-            if let Some(rest) = line.drop_prefix(&[s, &b":"[..]].concat()) {
-                line_syntax = rels;
-                line = rest;
-                break;
-            }
-        }
-
-        inputs.push(IgnorePattern::new(
-            parse_pattern_syntax(line_syntax).map_err(|e| match e {
-                PatternError::UnsupportedSyntax(syntax) => {
-                    PatternError::UnsupportedSyntaxInFile(
-                        syntax,
-                        file_path.to_string_lossy().into(),
-                        line_number,
-                    )
-                }
-                _ => e,
-            })?,
-            line,
-            file_path,
-        ));
     }
     Ok((inputs, warnings))
 }
 
 pub fn read_pattern_file(
     file_path: &Path,
     warn: bool,
     inspect_pattern_bytes: &mut impl FnMut(&Path, &[u8]),
 ) -> Result<(Vec<IgnorePattern>, Vec<PatternFileWarning>), PatternError> {
     match std::fs::read(file_path) {
         Ok(contents) => {
             inspect_pattern_bytes(file_path, &contents);
-            parse_pattern_file_contents(&contents, file_path, None, warn)
+            parse_pattern_file_contents(&contents, file_path, None, warn, true)
         }
         Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok((
             vec![],
             vec![PatternFileWarning::NoSuchFile(file_path.to_owned())],
         )),
         Err(e) => Err(e.into()),
     }
 }
 
 /// Represents an entry in an "ignore" file.
 #[derive(Debug, Eq, PartialEq, Clone)]
 pub struct IgnorePattern {
     pub syntax: PatternSyntax,
     pub pattern: Vec<u8>,
     pub source: PathBuf,
 }
 
 impl IgnorePattern {
     pub fn new(syntax: PatternSyntax, pattern: &[u8], source: &Path) -> Self {
         Self {
             syntax,
             pattern: pattern.to_owned(),
             source: source.to_owned(),
         }
     }
+
+    pub fn to_relative(self) -> Self {
+        let Self {
+            syntax,
+            pattern,
+            source,
+        } = self;
+        Self {
+            syntax: match syntax {
+                PatternSyntax::Regexp => PatternSyntax::RelRegexp,
+                PatternSyntax::Glob => PatternSyntax::RelGlob,
+                x => x,
+            },
+            pattern,
+            source,
+        }
+    }
 }
 
 pub type PatternResult<T> = Result<T, PatternError>;
 
 /// Wrapper for `read_pattern_file` that also recursively expands `include:`
 /// and `subinclude:` patterns.
 ///
 /// The former are expanded in place, while `PatternSyntax::ExpandedSubInclude`
 /// is used for the latter to form a tree of patterns.
 pub fn get_patterns_from_file(
     pattern_file: &Path,
     root_dir: &Path,
     inspect_pattern_bytes: &mut impl FnMut(&Path, &[u8]),
 ) -> PatternResult<(Vec<IgnorePattern>, Vec<PatternFileWarning>)> {
     let (patterns, mut warnings) =
         read_pattern_file(pattern_file, true, inspect_pattern_bytes)?;
     let patterns = patterns
         .into_iter()
         .flat_map(|entry| -> PatternResult<_> {
             Ok(match &entry.syntax {
                 PatternSyntax::Include => {
                     let inner_include =
                         root_dir.join(get_path_from_bytes(&entry.pattern));
                     let (inner_pats, inner_warnings) = get_patterns_from_file(
                         &inner_include,
                         root_dir,
                         inspect_pattern_bytes,
                     )?;
                     warnings.extend(inner_warnings);
                     inner_pats
                 }
                 PatternSyntax::SubInclude => {
                     let mut sub_include = SubInclude::new(
                         root_dir,
                         &entry.pattern,
                         &entry.source,
                     )?;
                     let (inner_patterns, inner_warnings) =
                         get_patterns_from_file(
                             &sub_include.path,
                             &sub_include.root,
                             inspect_pattern_bytes,
                         )?;
                     sub_include.included_patterns = inner_patterns;
                     warnings.extend(inner_warnings);
                     vec![IgnorePattern {
                         syntax: PatternSyntax::ExpandedSubInclude(Box::new(
                             sub_include,
                         )),
                         ..entry
                     }]
                 }
                 _ => vec![entry],
             })
         })
         .flatten()
         .collect();
 
     Ok((patterns, warnings))
 }
 
 /// Holds all the information needed to handle a `subinclude:` pattern.
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub struct SubInclude {
     /// Will be used for repository (hg) paths that start with this prefix.
     /// It is relative to the current working directory, so comparing against
     /// repository paths is painless.
     pub prefix: HgPathBuf,
     /// The file itself, containing the patterns
     pub path: PathBuf,
     /// Folder in the filesystem where this it applies
     pub root: PathBuf,
 
     pub included_patterns: Vec<IgnorePattern>,
 }
 
 impl SubInclude {
     pub fn new(
         root_dir: &Path,
         pattern: &[u8],
         source: &Path,
     ) -> Result<SubInclude, HgPathError> {
         let normalized_source =
             normalize_path_bytes(&get_bytes_from_path(source));
 
         let source_root = get_path_from_bytes(&normalized_source);
         let source_root =
             source_root.parent().unwrap_or_else(|| source_root.deref());
 
         let path = source_root.join(get_path_from_bytes(pattern));
         let new_root = path.parent().unwrap_or_else(|| path.deref());
 
         let prefix = canonical_path(root_dir, root_dir, new_root)?;
 
         Ok(Self {
             prefix: path_to_hg_path_buf(prefix).map(|mut p| {
                 if !p.is_empty() {
                     p.push_byte(b'/');
                 }
                 p
             })?,
             path: path.to_owned(),
             root: new_root.to_owned(),
             included_patterns: Vec::new(),
         })
     }
 }
 
 /// Separate and pre-process subincludes from other patterns for the "ignore"
 /// phase.
 pub fn filter_subincludes(
     ignore_patterns: Vec<IgnorePattern>,
 ) -> Result<(Vec<SubInclude>, Vec<IgnorePattern>), HgPathError> {
     let mut subincludes = vec![];
     let mut others = vec![];
 
     for pattern in ignore_patterns {
         if let PatternSyntax::ExpandedSubInclude(sub_include) = pattern.syntax
         {
             subincludes.push(*sub_include);
         } else {
             others.push(pattern)
         }
     }
     Ok((subincludes, others))
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
 
     #[test]
     fn escape_pattern_test() {
         let untouched =
             br#"!"%',/0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ_`abcdefghijklmnopqrstuvwxyz"#;
         assert_eq!(escape_pattern(untouched), untouched.to_vec());
         // All escape codes
         assert_eq!(
             escape_pattern(br#"()[]{}?*+-|^$\\.&~# \t\n\r\v\f"#),
             br#"\(\)\[\]\{\}\?\*\+\-\|\^\$\\\\\.\&\~\#\ \\t\\n\\r\\v\\f"#
                 .to_vec()
         );
     }
 
     #[test]
     fn glob_test() {
         assert_eq!(glob_to_re(br#"?"#), br#"."#);
         assert_eq!(glob_to_re(br#"*"#), br#"[^/]*"#);
         assert_eq!(glob_to_re(br#"**"#), br#".*"#);
         assert_eq!(glob_to_re(br#"**/a"#), br#"(?:.*/)?a"#);
         assert_eq!(glob_to_re(br#"a/**/b"#), br#"a/(?:.*/)?b"#);
         assert_eq!(glob_to_re(br#"[a*?!^][^b][!c]"#), br#"[a*?!^][\^b][^c]"#);
         assert_eq!(glob_to_re(br#"{a,b}"#), br#"(?:a|b)"#);
         assert_eq!(glob_to_re(br#".\*\?"#), br#"\.\*\?"#);
     }
 
     #[test]
     fn test_parse_pattern_file_contents() {
         let lines = b"syntax: glob\n*.elc";
 
         assert_eq!(
             parse_pattern_file_contents(
                 lines,
                 Path::new("file_path"),
                 None,
-                false
+                false,
+                true,
             )
             .unwrap()
             .0,
             vec![IgnorePattern::new(
                 PatternSyntax::RelGlob,
                 b"*.elc",
                 Path::new("file_path")
             )],
         );
 
         let lines = b"syntax: include\nsyntax: glob";
 
         assert_eq!(
             parse_pattern_file_contents(
                 lines,
                 Path::new("file_path"),
                 None,
-                false
+                false,
+                true,
             )
             .unwrap()
             .0,
             vec![]
         );
         let lines = b"glob:**.o";
         assert_eq!(
             parse_pattern_file_contents(
                 lines,
                 Path::new("file_path"),
                 None,
-                false
+                false,
+                true,
             )
             .unwrap()
             .0,
             vec![IgnorePattern::new(
                 PatternSyntax::RelGlob,
                 b"**.o",
                 Path::new("file_path")
             )]
         );
     }
 
     #[test]
     fn test_build_single_regex() {
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RelGlob,
                 b"rust/target/",
                 Path::new("")
             ))
             .unwrap(),
             Some(br"(?:.*/)?rust/target(?:/|$)".to_vec()),
         );
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::Regexp,
                 br"rust/target/\d+",
                 Path::new("")
             ))
             .unwrap(),
             Some(br"rust/target/\d+".to_vec()),
         );
     }
 
     #[test]
     fn test_build_single_regex_shortcut() {
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RootGlob,
                 b"",
                 Path::new("")
             ))
             .unwrap(),
             None,
         );
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RootGlob,
                 b"whatever",
                 Path::new("")
             ))
             .unwrap(),
             None,
         );
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RootGlob,
                 b"*.o",
                 Path::new("")
             ))
             .unwrap(),
             Some(br"[^/]*\.o(?:/|$)".to_vec()),
         );
     }
 
     #[test]
     fn test_build_single_relregex() {
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RelRegexp,
                 b"^ba{2}r",
                 Path::new("")
             ))
             .unwrap(),
             Some(b"^ba{2}r".to_vec()),
         );
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RelRegexp,
                 b"ba{2}r",
                 Path::new("")
             ))
             .unwrap(),
             Some(b".*ba{2}r".to_vec()),
         );
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RelRegexp,
                 b"(?ia)ba{2}r",
                 Path::new("")
             ))
             .unwrap(),
             Some(b"(?ia:.*ba{2}r)".to_vec()),
         );
         assert_eq!(
             build_single_regex(&IgnorePattern::new(
                 PatternSyntax::RelRegexp,
                 b"(?ia)^ba{2}r",
                 Path::new("")
             ))
             .unwrap(),
             Some(b"(?ia:^ba{2}r)".to_vec()),
         );
     }
 }

rust/hg-core/src/narrow.rs

@@ -1,113 +1,115 b''
 use std::path::Path;
 
 use crate::{
     errors::HgError,
     exit_codes,
     filepatterns::parse_pattern_file_contents,
     matchers::{
         AlwaysMatcher, DifferenceMatcher, IncludeMatcher, Matcher,
         NeverMatcher,
     },
     repo::Repo,
     requirements::NARROW_REQUIREMENT,
     sparse::{self, SparseConfigError, SparseWarning},
 };
 
 /// The file in .hg/store/ that indicates which paths exit in the store
 const FILENAME: &str = "narrowspec";
 /// The file in .hg/ that indicates which paths exit in the dirstate
 const DIRSTATE_FILENAME: &str = "narrowspec.dirstate";
 
 /// Pattern prefixes that are allowed in narrow patterns. This list MUST
 /// only contain patterns that are fast and safe to evaluate. Keep in mind
 /// that patterns are supplied by clients and executed on remote servers
 /// as part of wire protocol commands. That means that changes to this
 /// data structure influence the wire protocol and should not be taken
 /// lightly - especially removals.
 const VALID_PREFIXES: [&str; 2] = ["path:", "rootfilesin:"];
 
 /// Return the matcher for the current narrow spec, and all configuration
 /// warnings to display.
 pub fn matcher(
     repo: &Repo,
 ) -> Result<(Box<dyn Matcher + Sync>, Vec<SparseWarning>), SparseConfigError> {
     let mut warnings = vec![];
     if !repo.requirements().contains(NARROW_REQUIREMENT) {
         return Ok((Box::new(AlwaysMatcher), warnings));
     }
     // Treat "narrowspec does not exist" the same as "narrowspec file exists
     // and is empty".
     let store_spec = repo.store_vfs().try_read(FILENAME)?.unwrap_or_default();
     let working_copy_spec = repo
         .hg_vfs()
         .try_read(DIRSTATE_FILENAME)?
         .unwrap_or_default();
     if store_spec != working_copy_spec {
         return Err(HgError::abort(
             "abort: working copy's narrowspec is stale",
             exit_codes::STATE_ERROR,
             Some("run 'hg tracked --update-working-copy'".into()),
         )
         .into());
     }
 
     let config = sparse::parse_config(
         &store_spec,
         sparse::SparseConfigContext::Narrow,
     )?;
 
     warnings.extend(config.warnings);
 
     if !config.profiles.is_empty() {
         // TODO (from Python impl) maybe do something with profiles?
         return Err(SparseConfigError::IncludesInNarrow);
     }
     validate_patterns(&config.includes)?;
     validate_patterns(&config.excludes)?;
 
     if config.includes.is_empty() {
         return Ok((Box::new(NeverMatcher), warnings));
     }
 
     let (patterns, subwarnings) = parse_pattern_file_contents(
         &config.includes,
         Path::new(""),
         None,
         false,
+        true,
     )?;
     warnings.extend(subwarnings.into_iter().map(From::from));
 
     let mut m: Box<dyn Matcher + Sync> =
         Box::new(IncludeMatcher::new(patterns)?);
 
     let (patterns, subwarnings) = parse_pattern_file_contents(
         &config.excludes,
         Path::new(""),
         None,
         false,
+        true,
     )?;
     if !patterns.is_empty() {
         warnings.extend(subwarnings.into_iter().map(From::from));
         let exclude_matcher = Box::new(IncludeMatcher::new(patterns)?);
         m = Box::new(DifferenceMatcher::new(m, exclude_matcher));
     }
 
     Ok((m, warnings))
 }
 
 fn validate_patterns(patterns: &[u8]) -> Result<(), SparseConfigError> {
     for pattern in patterns.split(|c| *c == b'\n') {
         if pattern.is_empty() {
             continue;
         }
         for prefix in VALID_PREFIXES.iter() {
             if pattern.starts_with(prefix.as_bytes()) {
                 return Ok(());
             }
         }
         return Err(SparseConfigError::InvalidNarrowPrefix(
             pattern.to_owned(),
         ));
     }
     Ok(())
 }

rust/hg-core/src/sparse.rs

@@ -1,339 +1,341 b''
 use std::{collections::HashSet, path::Path};
 
 use format_bytes::{write_bytes, DisplayBytes};
 
 use crate::{
     errors::HgError,
     filepatterns::parse_pattern_file_contents,
     matchers::{
         AlwaysMatcher, DifferenceMatcher, IncludeMatcher, Matcher,
         UnionMatcher,
     },
     operations::cat,
     repo::Repo,
     requirements::SPARSE_REQUIREMENT,
     utils::{hg_path::HgPath, SliceExt},
     IgnorePattern, PatternError, PatternFileWarning, PatternSyntax, Revision,
     NULL_REVISION,
 };
 
 /// Command which is triggering the config read
 #[derive(Copy, Clone, Debug)]
 pub enum SparseConfigContext {
     Sparse,
     Narrow,
 }
 
 impl DisplayBytes for SparseConfigContext {
     fn display_bytes(
         &self,
         output: &mut dyn std::io::Write,
     ) -> std::io::Result<()> {
         match self {
             SparseConfigContext::Sparse => write_bytes!(output, b"sparse"),
             SparseConfigContext::Narrow => write_bytes!(output, b"narrow"),
         }
     }
 }
 
 /// Possible warnings when reading sparse configuration
 #[derive(Debug, derive_more::From)]
 pub enum SparseWarning {
     /// Warns about improper paths that start with "/"
     RootWarning {
         context: SparseConfigContext,
         line: Vec<u8>,
     },
     /// Warns about a profile missing from the given changelog revision
     ProfileNotFound { profile: Vec<u8>, rev: Revision },
     #[from]
     Pattern(PatternFileWarning),
 }
 
 /// Parsed sparse config
 #[derive(Debug, Default)]
 pub struct SparseConfig {
     // Line-separated
     pub(crate) includes: Vec<u8>,
     // Line-separated
     pub(crate) excludes: Vec<u8>,
     pub(crate) profiles: HashSet<Vec<u8>>,
     pub(crate) warnings: Vec<SparseWarning>,
 }
 
 /// All possible errors when reading sparse/narrow config
 #[derive(Debug, derive_more::From)]
 pub enum SparseConfigError {
     IncludesAfterExcludes {
         context: SparseConfigContext,
     },
     EntryOutsideSection {
         context: SparseConfigContext,
         line: Vec<u8>,
     },
     /// Narrow config does not support '%include' directives
     IncludesInNarrow,
     /// An invalid pattern prefix was given to the narrow spec. Includes the
     /// entire pattern for context.
     InvalidNarrowPrefix(Vec<u8>),
     #[from]
     HgError(HgError),
     #[from]
     PatternError(PatternError),
 }
 
 /// Parse sparse config file content.
 pub(crate) fn parse_config(
     raw: &[u8],
     context: SparseConfigContext,
 ) -> Result<SparseConfig, SparseConfigError> {
     let mut includes = vec![];
     let mut excludes = vec![];
     let mut profiles = HashSet::new();
     let mut warnings = vec![];
 
     #[derive(PartialEq, Eq)]
     enum Current {
         Includes,
         Excludes,
         None,
     }
 
     let mut current = Current::None;
     let mut in_section = false;
 
     for line in raw.split(|c| *c == b'\n') {
         let line = line.trim();
         if line.is_empty() || line[0] == b'#' {
             // empty or comment line, skip
             continue;
         }
         if line.starts_with(b"%include ") {
             let profile = line[b"%include ".len()..].trim();
             if !profile.is_empty() {
                 profiles.insert(profile.into());
             }
         } else if line == b"[include]" {
             if in_section && current == Current::Includes {
                 return Err(SparseConfigError::IncludesAfterExcludes {
                     context,
                 });
             }
             in_section = true;
             current = Current::Includes;
             continue;
         } else if line == b"[exclude]" {
             in_section = true;
             current = Current::Excludes;
         } else {
             if current == Current::None {
                 return Err(SparseConfigError::EntryOutsideSection {
                     context,
                     line: line.into(),
                 });
             }
             if line.trim().starts_with(b"/") {
                 warnings.push(SparseWarning::RootWarning {
                     context,
                     line: line.into(),
                 });
                 continue;
             }
             match current {
                 Current::Includes => {
                     includes.push(b'\n');
                     includes.extend(line.iter());
                 }
                 Current::Excludes => {
                     excludes.push(b'\n');
                     excludes.extend(line.iter());
                 }
                 Current::None => unreachable!(),
             }
         }
     }
 
     Ok(SparseConfig {
         includes,
         excludes,
         profiles,
         warnings,
     })
 }
 
 fn read_temporary_includes(
     repo: &Repo,
 ) -> Result<Vec<Vec<u8>>, SparseConfigError> {
     let raw = repo.hg_vfs().try_read("tempsparse")?.unwrap_or_default();
     if raw.is_empty() {
         return Ok(vec![]);
     }
     Ok(raw.split(|c| *c == b'\n').map(ToOwned::to_owned).collect())
 }
 
 /// Obtain sparse checkout patterns for the given revision
 fn patterns_for_rev(
     repo: &Repo,
     rev: Revision,
 ) -> Result<Option<SparseConfig>, SparseConfigError> {
     if !repo.has_sparse() {
         return Ok(None);
     }
     let raw = repo.hg_vfs().try_read("sparse")?.unwrap_or_default();
 
     if raw.is_empty() {
         return Ok(None);
     }
 
     let mut config = parse_config(&raw, SparseConfigContext::Sparse)?;
 
     if !config.profiles.is_empty() {
         let mut profiles: Vec<Vec<u8>> = config.profiles.into_iter().collect();
         let mut visited = HashSet::new();
 
         while let Some(profile) = profiles.pop() {
             if visited.contains(&profile) {
                 continue;
             }
             visited.insert(profile.to_owned());
 
             let output =
                 cat(repo, &rev.to_string(), vec![HgPath::new(&profile)])
                     .map_err(|_| {
                         HgError::corrupted(
                             "dirstate points to non-existent parent node"
                                 .to_string(),
                         )
                     })?;
             if output.results.is_empty() {
                 config.warnings.push(SparseWarning::ProfileNotFound {
                     profile: profile.to_owned(),
                     rev,
                 })
             }
 
             let subconfig = parse_config(
                 &output.results[0].1,
                 SparseConfigContext::Sparse,
             )?;
             if !subconfig.includes.is_empty() {
                 config.includes.push(b'\n');
                 config.includes.extend(&subconfig.includes);
             }
             if !subconfig.includes.is_empty() {
                 config.includes.push(b'\n');
                 config.excludes.extend(&subconfig.excludes);
             }
             config.warnings.extend(subconfig.warnings.into_iter());
             profiles.extend(subconfig.profiles.into_iter());
         }
 
         config.profiles = visited;
     }
 
     if !config.includes.is_empty() {
         config.includes.extend(b"\n.hg*");
     }
 
     Ok(Some(config))
 }
 
 /// Obtain a matcher for sparse working directories.
 pub fn matcher(
     repo: &Repo,
 ) -> Result<(Box<dyn Matcher + Sync>, Vec<SparseWarning>), SparseConfigError> {
     let mut warnings = vec![];
     if !repo.requirements().contains(SPARSE_REQUIREMENT) {
         return Ok((Box::new(AlwaysMatcher), warnings));
     }
 
     let parents = repo.dirstate_parents()?;
     let mut revs = vec![];
     let p1_rev =
         repo.changelog()?
             .rev_from_node(parents.p1.into())
             .map_err(|_| {
                 HgError::corrupted(
                     "dirstate points to non-existent parent node".to_string(),
                 )
             })?;
     if p1_rev != NULL_REVISION {
         revs.push(p1_rev)
     }
     let p2_rev =
         repo.changelog()?
             .rev_from_node(parents.p2.into())
             .map_err(|_| {
                 HgError::corrupted(
                     "dirstate points to non-existent parent node".to_string(),
                 )
             })?;
     if p2_rev != NULL_REVISION {
         revs.push(p2_rev)
     }
     let mut matchers = vec![];
 
     for rev in revs.iter() {
         let config = patterns_for_rev(repo, *rev);
         if let Ok(Some(config)) = config {
             warnings.extend(config.warnings);
             let mut m: Box<dyn Matcher + Sync> = Box::new(AlwaysMatcher);
             if !config.includes.is_empty() {
                 let (patterns, subwarnings) = parse_pattern_file_contents(
                     &config.includes,
                     Path::new(""),
-                    Some(b"glob:".as_ref()),
+                    Some(PatternSyntax::Glob),
+                    false,
                     false,
                 )?;
                 warnings.extend(subwarnings.into_iter().map(From::from));
                 m = Box::new(IncludeMatcher::new(patterns)?);
             }
             if !config.excludes.is_empty() {
                 let (patterns, subwarnings) = parse_pattern_file_contents(
                     &config.excludes,
                     Path::new(""),
-                    Some(b"glob:".as_ref()),
+                    Some(PatternSyntax::Glob),
+                    false,
                     false,
                 )?;
                 warnings.extend(subwarnings.into_iter().map(From::from));
                 m = Box::new(DifferenceMatcher::new(
                     m,
                     Box::new(IncludeMatcher::new(patterns)?),
                 ));
             }
             matchers.push(m);
         }
     }
     let result: Box<dyn Matcher + Sync> = match matchers.len() {
         0 => Box::new(AlwaysMatcher),
         1 => matchers.pop().expect("1 is equal to 0"),
         _ => Box::new(UnionMatcher::new(matchers)),
     };
 
     let matcher =
         force_include_matcher(result, &read_temporary_includes(repo)?)?;
     Ok((matcher, warnings))
 }
 
 /// Returns a matcher that returns true for any of the forced includes before
 /// testing against the actual matcher
 fn force_include_matcher(
     result: Box<dyn Matcher + Sync>,
     temp_includes: &[Vec<u8>],
 ) -> Result<Box<dyn Matcher + Sync>, PatternError> {
     if temp_includes.is_empty() {
         return Ok(result);
     }
     let forced_include_matcher = IncludeMatcher::new(
         temp_includes
             .iter()
             .map(|include| {
                 IgnorePattern::new(PatternSyntax::Path, include, Path::new(""))
             })
             .collect(),
     )?;
     Ok(Box::new(UnionMatcher::new(vec![
         Box::new(forced_include_matcher),
         result,
     ])))
 }