##// END OF EJS Templates
rust-revlog: introduce an `options` module...
Raphaël Gomès -
r53053:039b7cae default
parent child Browse files
Show More
@@ -0,0 +1,393
1 //! Helpers for the revlog config and opening options
2
3 use std::collections::HashSet;
4
5 use crate::{
6 config::{Config, ResourceProfileValue},
7 errors::HgError,
8 requirements::{
9 CHANGELOGV2_REQUIREMENT, GENERALDELTA_REQUIREMENT, NARROW_REQUIREMENT,
10 NODEMAP_REQUIREMENT, REVLOGV1_REQUIREMENT, REVLOGV2_REQUIREMENT,
11 SPARSEREVLOG_REQUIREMENT,
12 },
13 };
14
15 use super::{compression::CompressionConfig, RevlogType};
16
17 const DEFAULT_CHUNK_CACHE_SIZE: u64 = 65536;
18 const DEFAULT_SPARSE_READ_DENSITY_THRESHOLD: f64 = 0.50;
19 const DEFAULT_SPARSE_READ_MIN_GAP_SIZE: u64 = 262144;
20
21 /// The known revlog versions and their options
22 #[derive(Debug, Copy, Clone, PartialEq)]
23 pub enum RevlogVersionOptions {
24 V0,
25 V1 { general_delta: bool, inline: bool },
26 V2,
27 ChangelogV2 { compute_rank: bool },
28 }
29
30 /// Options to govern how a revlog should be opened, usually from the
31 /// repository configuration or requirements.
32 #[derive(Debug, Copy, Clone)]
33 pub struct RevlogOpenOptions {
34 /// The revlog version, along with any option specific to this version
35 pub version: RevlogVersionOptions,
36 /// Whether the revlog uses a persistent nodemap.
37 pub use_nodemap: bool,
38 pub delta_config: RevlogDeltaConfig,
39 pub data_config: RevlogDataConfig,
40 pub feature_config: RevlogFeatureConfig,
41 }
42
43 #[cfg(test)]
44 impl Default for RevlogOpenOptions {
45 fn default() -> Self {
46 Self {
47 version: RevlogVersionOptions::V1 {
48 general_delta: true,
49 inline: false,
50 },
51 use_nodemap: true,
52 data_config: Default::default(),
53 delta_config: Default::default(),
54 feature_config: Default::default(),
55 }
56 }
57 }
58
59 impl RevlogOpenOptions {
60 pub fn new(
61 inline: bool,
62 data_config: RevlogDataConfig,
63 delta_config: RevlogDeltaConfig,
64 feature_config: RevlogFeatureConfig,
65 ) -> Self {
66 Self {
67 version: RevlogVersionOptions::V1 {
68 general_delta: data_config.general_delta,
69 inline,
70 },
71 use_nodemap: false,
72 data_config,
73 delta_config,
74 feature_config,
75 }
76 }
77
78 pub fn index_header(&self) -> super::index::IndexHeader {
79 super::index::IndexHeader {
80 header_bytes: match self.version {
81 RevlogVersionOptions::V0 => [0, 0, 0, 0],
82 RevlogVersionOptions::V1 {
83 general_delta,
84 inline,
85 } => [
86 0,
87 if general_delta && inline {
88 3
89 } else if general_delta {
90 2
91 } else {
92 u8::from(inline)
93 },
94 0,
95 1,
96 ],
97 RevlogVersionOptions::V2 => 0xDEADu32.to_be_bytes(),
98 RevlogVersionOptions::ChangelogV2 { compute_rank: _ } => {
99 0xD34Du32.to_be_bytes()
100 }
101 },
102 }
103 }
104 }
105
106 #[derive(Debug, Clone, Copy, PartialEq)]
107 /// Holds configuration values about how the revlog data is read
108 pub struct RevlogDataConfig {
109 /// Should we try to open the "pending" version of the revlog
110 pub try_pending: bool,
111 /// Should we try to open the "split" version of the revlog
112 pub try_split: bool,
113 /// When True, `indexfile` should be opened with `checkambig=True` at
114 /// writing time, to avoid file stat ambiguity
115 pub check_ambig: bool,
116 /// If true, use mmap instead of reading to deal with large indexes
117 pub mmap_large_index: bool,
118 /// How much data is considered large
119 pub mmap_index_threshold: Option<u64>,
120 /// How much data to read and cache into the raw revlog data cache
121 pub chunk_cache_size: u64,
122 /// The size of the uncompressed cache compared to the largest revision
123 /// seen
124 pub uncompressed_cache_factor: Option<f64>,
125 /// The number of chunks cached
126 pub uncompressed_cache_count: Option<u64>,
127 /// Allow sparse reading of the revlog data
128 pub with_sparse_read: bool,
129 /// Minimal density of a sparse read chunk
130 pub sr_density_threshold: f64,
131 /// Minimal size of the data we skip when performing sparse reads
132 pub sr_min_gap_size: u64,
133 /// Whether deltas are encoded against arbitrary bases
134 pub general_delta: bool,
135 }
136
137 impl RevlogDataConfig {
138 pub fn new(
139 config: &Config,
140 requirements: &HashSet<String>,
141 ) -> Result<Self, HgError> {
142 let mut data_config = Self::default();
143 if let Some(chunk_cache_size) =
144 config.get_byte_size(b"format", b"chunkcachesize")?
145 {
146 data_config.chunk_cache_size = chunk_cache_size;
147 }
148
149 let memory_profile = config.get_resource_profile(Some("memory"));
150 if memory_profile.value >= ResourceProfileValue::Medium {
151 data_config.uncompressed_cache_count = Some(10_000);
152 data_config.uncompressed_cache_factor = Some(4.0);
153 if memory_profile.value >= ResourceProfileValue::High {
154 data_config.uncompressed_cache_factor = Some(10.0)
155 }
156 }
157
158 if let Some(mmap_index_threshold) = config
159 .get_byte_size(b"storage", b"revlog.mmap.index:size-threshold")?
160 {
161 data_config.mmap_index_threshold = Some(mmap_index_threshold);
162 }
163
164 let with_sparse_read =
165 config.get_bool(b"experimental", b"sparse-read")?;
166 if let Some(sr_density_threshold) = config
167 .get_f64(b"experimental", b"sparse-read.density-threshold")?
168 {
169 data_config.sr_density_threshold = sr_density_threshold;
170 }
171 data_config.with_sparse_read = with_sparse_read;
172 if let Some(sr_min_gap_size) = config
173 .get_byte_size(b"experimental", b"sparse-read.min-gap-size")?
174 {
175 data_config.sr_min_gap_size = sr_min_gap_size;
176 }
177
178 data_config.with_sparse_read =
179 requirements.contains(SPARSEREVLOG_REQUIREMENT);
180
181 Ok(data_config)
182 }
183 }
184
185 impl Default for RevlogDataConfig {
186 fn default() -> Self {
187 Self {
188 chunk_cache_size: DEFAULT_CHUNK_CACHE_SIZE,
189 sr_density_threshold: DEFAULT_SPARSE_READ_DENSITY_THRESHOLD,
190 sr_min_gap_size: DEFAULT_SPARSE_READ_MIN_GAP_SIZE,
191 try_pending: Default::default(),
192 try_split: Default::default(),
193 check_ambig: Default::default(),
194 mmap_large_index: Default::default(),
195 mmap_index_threshold: Default::default(),
196 uncompressed_cache_factor: Default::default(),
197 uncompressed_cache_count: Default::default(),
198 with_sparse_read: Default::default(),
199 general_delta: Default::default(),
200 }
201 }
202 }
203
204 #[derive(Debug, Clone, Copy, PartialEq)]
205 /// Holds configuration values about how new deltas are computed.
206 ///
207 /// Some attributes are duplicated from [`RevlogDataConfig`] to help having
208 /// each object self contained.
209 pub struct RevlogDeltaConfig {
210 /// Whether deltas can be encoded against arbitrary bases
211 pub general_delta: bool,
212 /// Allow sparse writing of the revlog data
213 pub sparse_revlog: bool,
214 /// Maximum length of a delta chain
215 pub max_chain_len: Option<u64>,
216 /// Maximum distance between a delta chain's start and end
217 pub max_deltachain_span: Option<u64>,
218 /// If `upper_bound_comp` is not None, this is the expected maximal
219 /// gain from compression for the data content
220 pub upper_bound_comp: Option<f64>,
221 /// Should we try a delta against both parents
222 pub delta_both_parents: bool,
223 /// Test delta base candidate groups by chunks of this maximal size
224 pub candidate_group_chunk_size: u64,
225 /// Should we display debug information about delta computation
226 pub debug_delta: bool,
227 /// Trust incoming deltas by default
228 pub lazy_delta: bool,
229 /// Trust the base of incoming deltas by default
230 pub lazy_delta_base: bool,
231 }
232
233 impl RevlogDeltaConfig {
234 pub fn new(
235 config: &Config,
236 requirements: &HashSet<String>,
237 revlog_type: RevlogType,
238 ) -> Result<Self, HgError> {
239 let mut delta_config = Self {
240 delta_both_parents: config
241 .get_option_no_default(
242 b"storage",
243 b"revlog.optimize-delta-parent-choice",
244 )?
245 .unwrap_or(true),
246 candidate_group_chunk_size: config
247 .get_u64(
248 b"storage",
249 b"revlog.delta-parent-search.candidate-group-chunk-size",
250 )?
251 .unwrap_or_default(),
252 ..Default::default()
253 };
254
255 delta_config.debug_delta =
256 config.get_bool(b"debug", b"revlog.debug-delta")?;
257
258 delta_config.general_delta =
259 requirements.contains(GENERALDELTA_REQUIREMENT);
260
261 let lazy_delta =
262 config.get_bool(b"storage", b"revlog.reuse-external-delta")?;
263
264 if revlog_type == RevlogType::Manifestlog {
265 // upper bound of what we expect from compression
266 // (real life value seems to be 3)
267 delta_config.upper_bound_comp = Some(3.0)
268 }
269
270 let mut lazy_delta_base = false;
271 if lazy_delta {
272 lazy_delta_base = match config.get_option_no_default(
273 b"storage",
274 b"revlog.reuse-external-delta-parent",
275 )? {
276 Some(base) => base,
277 None => config.get_bool(b"format", b"generaldelta")?,
278 };
279 }
280 delta_config.lazy_delta = lazy_delta;
281 delta_config.lazy_delta_base = lazy_delta_base;
282
283 delta_config.max_deltachain_span =
284 match config.get_i64(b"experimental", b"maxdeltachainspan")? {
285 Some(span) => {
286 if span < 0 {
287 None
288 } else {
289 Some(span as u64)
290 }
291 }
292 None => None,
293 };
294
295 delta_config.sparse_revlog =
296 requirements.contains(SPARSEREVLOG_REQUIREMENT);
297
298 delta_config.max_chain_len =
299 config.get_byte_size_no_default(b"format", b"maxchainlen")?;
300
301 Ok(delta_config)
302 }
303 }
304
305 impl Default for RevlogDeltaConfig {
306 fn default() -> Self {
307 Self {
308 delta_both_parents: true,
309 lazy_delta: true,
310 general_delta: Default::default(),
311 sparse_revlog: Default::default(),
312 max_chain_len: Default::default(),
313 max_deltachain_span: Default::default(),
314 upper_bound_comp: Default::default(),
315 candidate_group_chunk_size: Default::default(),
316 debug_delta: Default::default(),
317 lazy_delta_base: Default::default(),
318 }
319 }
320 }
321
322 #[derive(Debug, Default, Clone, Copy, PartialEq)]
323 /// Holds configuration values about the available revlog features
324 pub struct RevlogFeatureConfig {
325 /// The compression engine and its options
326 pub compression_engine: CompressionConfig,
327 /// Can we use censor on this revlog
328 pub censorable: bool,
329 /// Does this revlog use the "side data" feature
330 pub has_side_data: bool,
331 /// Might remove this configuration once the rank computation has no
332 /// impact
333 pub compute_rank: bool,
334 /// Parent order is supposed to be semantically irrelevant, so we
335 /// normally re-sort parents to ensure that the first parent is non-null,
336 /// if there is a non-null parent at all.
337 /// filelog abuses the parent order as a flag to mark some instances of
338 /// meta-encoded files, so allow it to disable this behavior.
339 pub canonical_parent_order: bool,
340 /// Can ellipsis commit be used
341 pub enable_ellipsis: bool,
342 }
343
344 impl RevlogFeatureConfig {
345 pub fn new(
346 config: &Config,
347 requirements: &HashSet<String>,
348 ) -> Result<Self, HgError> {
349 Ok(Self {
350 compression_engine: CompressionConfig::new(config, requirements)?,
351 enable_ellipsis: requirements.contains(NARROW_REQUIREMENT),
352 ..Default::default()
353 })
354 }
355 }
356
357 /// Return the default options for a revlog of `revlog_type` according to the
358 /// current config and requirements.
359 pub fn default_revlog_options(
360 config: &Config,
361 requirements: &HashSet<String>,
362 revlog_type: RevlogType,
363 ) -> Result<RevlogOpenOptions, HgError> {
364 let is_changelog = revlog_type == RevlogType::Changelog;
365 let version =
366 if is_changelog && requirements.contains(CHANGELOGV2_REQUIREMENT) {
367 let compute_rank = config
368 .get_bool(b"experimental", b"changelog-v2.compute-rank")?;
369 RevlogVersionOptions::ChangelogV2 { compute_rank }
370 } else if requirements.contains(REVLOGV2_REQUIREMENT) {
371 RevlogVersionOptions::V2
372 } else if requirements.contains(REVLOGV1_REQUIREMENT) {
373 RevlogVersionOptions::V1 {
374 general_delta: requirements.contains(GENERALDELTA_REQUIREMENT),
375 inline: !is_changelog,
376 }
377 } else {
378 RevlogVersionOptions::V0
379 };
380 Ok(RevlogOpenOptions {
381 version,
382 // We don't need to dance around the slow path like in the Python
383 // implementation since we know we have access to the fast code.
384 use_nodemap: requirements.contains(NODEMAP_REQUIREMENT),
385 delta_config: RevlogDeltaConfig::new(
386 config,
387 requirements,
388 revlog_type,
389 )?,
390 data_config: RevlogDataConfig::new(config, requirements)?,
391 feature_config: RevlogFeatureConfig::new(config, requirements)?,
392 })
393 }
@@ -7,6 +7,7
7
7
8 use crate::errors::HgError;
8 use crate::errors::HgError;
9 use crate::repo::Repo;
9 use crate::repo::Repo;
10 use crate::revlog::options::default_revlog_options;
10 use crate::revlog::Revlog;
11 use crate::revlog::Revlog;
11 use crate::{exit_codes, RevlogError, RevlogType};
12 use crate::{exit_codes, RevlogError, RevlogType};
12
13
@@ -31,7 +32,11 pub fn debug_data(
31 &repo.store_vfs(),
32 &repo.store_vfs(),
32 index_file,
33 index_file,
33 None,
34 None,
34 repo.default_revlog_options(RevlogType::Changelog)?,
35 default_revlog_options(
36 repo.config(),
37 repo.requirements(),
38 RevlogType::Changelog,
39 )?,
35 )?;
40 )?;
36 let rev =
41 let rev =
37 crate::revset::resolve_rev_number_or_hex_prefix(revset, &revlog)?;
42 crate::revset::resolve_rev_number_or_hex_prefix(revset, &revlog)?;
@@ -10,11 +10,8 use crate::errors::HgResultExt;
10 use crate::errors::{HgError, IoResultExt};
10 use crate::errors::{HgError, IoResultExt};
11 use crate::lock::{try_with_lock_no_wait, LockError};
11 use crate::lock::{try_with_lock_no_wait, LockError};
12 use crate::manifest::{Manifest, Manifestlog};
12 use crate::manifest::{Manifest, Manifestlog};
13 use crate::requirements::{
13 use crate::options::default_revlog_options;
14 CHANGELOGV2_REQUIREMENT, DIRSTATE_TRACKED_HINT_V1,
14 use crate::requirements::DIRSTATE_TRACKED_HINT_V1;
15 GENERALDELTA_REQUIREMENT, NODEMAP_REQUIREMENT, REVLOGV1_REQUIREMENT,
16 REVLOGV2_REQUIREMENT,
17 };
18 use crate::revlog::filelog::Filelog;
15 use crate::revlog::filelog::Filelog;
19 use crate::revlog::RevlogError;
16 use crate::revlog::RevlogError;
20 use crate::utils::debug::debug_wait_for_file_or_print;
17 use crate::utils::debug::debug_wait_for_file_or_print;
@@ -22,11 +19,10 use crate::utils::files::get_path_from_b
22 use crate::utils::hg_path::HgPath;
19 use crate::utils::hg_path::HgPath;
23 use crate::utils::SliceExt;
20 use crate::utils::SliceExt;
24 use crate::vfs::{is_dir, is_file, VfsImpl};
21 use crate::vfs::{is_dir, is_file, VfsImpl};
22 use crate::DirstateError;
25 use crate::{
23 use crate::{
26 exit_codes, requirements, NodePrefix, RevlogDataConfig, RevlogDeltaConfig,
24 exit_codes, requirements, NodePrefix, RevlogType, UncheckedRevision,
27 RevlogFeatureConfig, RevlogType, RevlogVersionOptions, UncheckedRevision,
28 };
25 };
29 use crate::{DirstateError, RevlogOpenOptions};
30 use std::cell::{Ref, RefCell, RefMut};
26 use std::cell::{Ref, RefCell, RefMut};
31 use std::collections::HashSet;
27 use std::collections::HashSet;
32 use std::io::Seek;
28 use std::io::Seek;
@@ -577,7 +573,11 impl Repo {
577 fn new_changelog(&self) -> Result<Changelog, HgError> {
573 fn new_changelog(&self) -> Result<Changelog, HgError> {
578 Changelog::open(
574 Changelog::open(
579 &self.store_vfs(),
575 &self.store_vfs(),
580 self.default_revlog_options(RevlogType::Changelog)?,
576 default_revlog_options(
577 self.config(),
578 self.requirements(),
579 RevlogType::Changelog,
580 )?,
581 )
581 )
582 }
582 }
583
583
@@ -592,7 +592,11 impl Repo {
592 fn new_manifestlog(&self) -> Result<Manifestlog, HgError> {
592 fn new_manifestlog(&self) -> Result<Manifestlog, HgError> {
593 Manifestlog::open(
593 Manifestlog::open(
594 &self.store_vfs(),
594 &self.store_vfs(),
595 self.default_revlog_options(RevlogType::Manifestlog)?,
595 default_revlog_options(
596 self.config(),
597 self.requirements(),
598 RevlogType::Manifestlog,
599 )?,
596 )
600 )
597 }
601 }
598
602
@@ -642,7 +646,11 impl Repo {
642 Filelog::open(
646 Filelog::open(
643 self,
647 self,
644 path,
648 path,
645 self.default_revlog_options(RevlogType::Filelog)?,
649 default_revlog_options(
650 self.config(),
651 self.requirements(),
652 RevlogType::Filelog,
653 )?,
646 )
654 )
647 }
655 }
648 /// Write to disk any updates that were made through `dirstate_map_mut`.
656 /// Write to disk any updates that were made through `dirstate_map_mut`.
@@ -792,50 +800,6 impl Repo {
792 Ok(())
800 Ok(())
793 }
801 }
794
802
795 pub fn default_revlog_options(
796 &self,
797 revlog_type: RevlogType,
798 ) -> Result<RevlogOpenOptions, HgError> {
799 let requirements = self.requirements();
800 let is_changelog = revlog_type == RevlogType::Changelog;
801 let version = if is_changelog
802 && requirements.contains(CHANGELOGV2_REQUIREMENT)
803 {
804 let compute_rank = self
805 .config()
806 .get_bool(b"experimental", b"changelog-v2.compute-rank")?;
807 RevlogVersionOptions::ChangelogV2 { compute_rank }
808 } else if requirements.contains(REVLOGV2_REQUIREMENT) {
809 RevlogVersionOptions::V2
810 } else if requirements.contains(REVLOGV1_REQUIREMENT) {
811 RevlogVersionOptions::V1 {
812 general_delta: requirements.contains(GENERALDELTA_REQUIREMENT),
813 inline: !is_changelog,
814 }
815 } else {
816 RevlogVersionOptions::V0
817 };
818 Ok(RevlogOpenOptions {
819 version,
820 // We don't need to dance around the slow path like in the Python
821 // implementation since we know we have access to the fast code.
822 use_nodemap: requirements.contains(NODEMAP_REQUIREMENT),
823 delta_config: RevlogDeltaConfig::new(
824 self.config(),
825 self.requirements(),
826 revlog_type,
827 )?,
828 data_config: RevlogDataConfig::new(
829 self.config(),
830 self.requirements(),
831 )?,
832 feature_config: RevlogFeatureConfig::new(
833 self.config(),
834 requirements,
835 )?,
836 })
837 }
838
839 pub fn node(&self, rev: UncheckedRevision) -> Option<crate::Node> {
803 pub fn node(&self, rev: UncheckedRevision) -> Option<crate::Node> {
840 self.changelog()
804 self.changelog()
841 .ok()
805 .ok()
@@ -14,7 +14,9 use crate::revlog::{Node, NodePrefix};
14 use crate::revlog::{Revlog, RevlogEntry, RevlogError};
14 use crate::revlog::{Revlog, RevlogEntry, RevlogError};
15 use crate::utils::hg_path::HgPath;
15 use crate::utils::hg_path::HgPath;
16 use crate::vfs::VfsImpl;
16 use crate::vfs::VfsImpl;
17 use crate::{Graph, GraphError, RevlogOpenOptions, UncheckedRevision};
17 use crate::{Graph, GraphError, UncheckedRevision};
18
19 use super::options::RevlogOpenOptions;
18
20
19 /// A specialized `Revlog` to work with changelog data format.
21 /// A specialized `Revlog` to work with changelog data format.
20 pub struct Changelog {
22 pub struct Changelog {
@@ -504,10 +506,7 fn unescape_extra(bytes: &[u8]) -> Vec<u
504 mod tests {
506 mod tests {
505 use super::*;
507 use super::*;
506 use crate::vfs::VfsImpl;
508 use crate::vfs::VfsImpl;
507 use crate::{
509 use crate::NULL_REVISION;
508 RevlogDataConfig, RevlogDeltaConfig, RevlogFeatureConfig,
509 NULL_REVISION,
510 };
511 use pretty_assertions::assert_eq;
510 use pretty_assertions::assert_eq;
512
511
513 #[test]
512 #[test]
@@ -571,18 +570,9 message",
571 };
570 };
572 std::fs::write(temp.path().join("foo.i"), b"").unwrap();
571 std::fs::write(temp.path().join("foo.i"), b"").unwrap();
573 std::fs::write(temp.path().join("foo.d"), b"").unwrap();
572 std::fs::write(temp.path().join("foo.d"), b"").unwrap();
574 let revlog = Revlog::open(
573 let revlog =
575 &vfs,
574 Revlog::open(&vfs, "foo.i", None, RevlogOpenOptions::default())
576 "foo.i",
575 .unwrap();
577 None,
578 RevlogOpenOptions::new(
579 false,
580 RevlogDataConfig::default(),
581 RevlogDeltaConfig::default(),
582 RevlogFeatureConfig::default(),
583 ),
584 )
585 .unwrap();
586
576
587 let changelog = Changelog { revlog };
577 let changelog = Changelog { revlog };
588 assert_eq!(
578 assert_eq!(
@@ -11,10 +11,11 use crate::utils::hg_path::HgPath;
11 use crate::utils::SliceExt;
11 use crate::utils::SliceExt;
12 use crate::Graph;
12 use crate::Graph;
13 use crate::GraphError;
13 use crate::GraphError;
14 use crate::RevlogOpenOptions;
15 use crate::UncheckedRevision;
14 use crate::UncheckedRevision;
16 use std::path::PathBuf;
15 use std::path::PathBuf;
17
16
17 use super::options::RevlogOpenOptions;
18
18 /// A specialized `Revlog` to work with file data logs.
19 /// A specialized `Revlog` to work with file data logs.
19 pub struct Filelog {
20 pub struct Filelog {
20 /// The generic `revlog` format.
21 /// The generic `revlog` format.
@@ -6,9 +6,9 use crate::revlog::{Revlog, RevlogError}
6 use crate::utils::hg_path::HgPath;
6 use crate::utils::hg_path::HgPath;
7 use crate::utils::SliceExt;
7 use crate::utils::SliceExt;
8 use crate::vfs::VfsImpl;
8 use crate::vfs::VfsImpl;
9 use crate::{
9 use crate::{Graph, GraphError, Revision, UncheckedRevision};
10 Graph, GraphError, Revision, RevlogOpenOptions, UncheckedRevision,
10
11 };
11 use super::options::RevlogOpenOptions;
12
12
13 /// A specialized `Revlog` to work with `manifest` data format.
13 /// A specialized `Revlog` to work with `manifest` data format.
14 pub struct Manifestlog {
14 pub struct Manifestlog {
@@ -9,18 +9,19 pub mod node;
9 pub mod nodemap;
9 pub mod nodemap;
10 mod nodemap_docket;
10 mod nodemap_docket;
11 pub mod path_encode;
11 pub mod path_encode;
12 use compression::{uncompressed_zstd_data, CompressionConfig};
12 use compression::uncompressed_zstd_data;
13 pub use node::{FromHexError, Node, NodePrefix};
13 pub use node::{FromHexError, Node, NodePrefix};
14 use options::RevlogOpenOptions;
14 pub mod changelog;
15 pub mod changelog;
15 pub mod compression;
16 pub mod compression;
16 pub mod file_io;
17 pub mod file_io;
17 pub mod filelog;
18 pub mod filelog;
18 pub mod index;
19 pub mod index;
19 pub mod manifest;
20 pub mod manifest;
21 pub mod options;
20 pub mod patch;
22 pub mod patch;
21
23
22 use std::borrow::Cow;
24 use std::borrow::Cow;
23 use std::collections::HashSet;
24 use std::io::Read;
25 use std::io::Read;
25 use std::ops::Deref;
26 use std::ops::Deref;
26 use std::path::Path;
27 use std::path::Path;
@@ -33,12 +34,8 use self::nodemap_docket::NodeMapDocket;
33 use super::index::Index;
34 use super::index::Index;
34 use super::index::INDEX_ENTRY_SIZE;
35 use super::index::INDEX_ENTRY_SIZE;
35 use super::nodemap::{NodeMap, NodeMapError};
36 use super::nodemap::{NodeMap, NodeMapError};
36 use crate::config::{Config, ResourceProfileValue};
37 use crate::errors::HgError;
37 use crate::errors::HgError;
38 use crate::exit_codes;
38 use crate::exit_codes;
39 use crate::requirements::{
40 GENERALDELTA_REQUIREMENT, NARROW_REQUIREMENT, SPARSEREVLOG_REQUIREMENT,
41 };
42 use crate::vfs::VfsImpl;
39 use crate::vfs::VfsImpl;
43
40
44 /// As noted in revlog.c, revision numbers are actually encoded in
41 /// As noted in revlog.c, revision numbers are actually encoded in
@@ -259,255 +256,6 impl TryFrom<usize> for RevlogType {
259 }
256 }
260 }
257 }
261
258
262 #[derive(Debug, Clone, Copy, PartialEq)]
263 /// Holds configuration values about how the revlog data is read
264 pub struct RevlogDataConfig {
265 /// Should we try to open the "pending" version of the revlog
266 pub try_pending: bool,
267 /// Should we try to open the "split" version of the revlog
268 pub try_split: bool,
269 /// When True, `indexfile` should be opened with `checkambig=True` at
270 /// writing time, to avoid file stat ambiguity
271 pub check_ambig: bool,
272 /// If true, use mmap instead of reading to deal with large indexes
273 pub mmap_large_index: bool,
274 /// How much data is considered large
275 pub mmap_index_threshold: Option<u64>,
276 /// How much data to read and cache into the raw revlog data cache
277 pub chunk_cache_size: u64,
278 /// The size of the uncompressed cache compared to the largest revision
279 /// seen
280 pub uncompressed_cache_factor: Option<f64>,
281 /// The number of chunks cached
282 pub uncompressed_cache_count: Option<u64>,
283 /// Allow sparse reading of the revlog data
284 pub with_sparse_read: bool,
285 /// Minimal density of a sparse read chunk
286 pub sr_density_threshold: f64,
287 /// Minimal size of the data we skip when performing sparse reads
288 pub sr_min_gap_size: u64,
289 /// Whether deltas are encoded against arbitrary bases
290 pub general_delta: bool,
291 }
292
293 impl RevlogDataConfig {
294 pub fn new(
295 config: &Config,
296 requirements: &HashSet<String>,
297 ) -> Result<Self, HgError> {
298 let mut data_config = Self::default();
299 if let Some(chunk_cache_size) =
300 config.get_byte_size(b"format", b"chunkcachesize")?
301 {
302 data_config.chunk_cache_size = chunk_cache_size;
303 }
304
305 let memory_profile = config.get_resource_profile(Some("memory"));
306 if memory_profile.value >= ResourceProfileValue::Medium {
307 data_config.uncompressed_cache_count = Some(10_000);
308 data_config.uncompressed_cache_factor = Some(4.0);
309 if memory_profile.value >= ResourceProfileValue::High {
310 data_config.uncompressed_cache_factor = Some(10.0)
311 }
312 }
313
314 if let Some(mmap_index_threshold) = config
315 .get_byte_size(b"storage", b"revlog.mmap.index:size-threshold")?
316 {
317 data_config.mmap_index_threshold = Some(mmap_index_threshold);
318 }
319
320 let with_sparse_read =
321 config.get_bool(b"experimental", b"sparse-read")?;
322 if let Some(sr_density_threshold) = config
323 .get_f64(b"experimental", b"sparse-read.density-threshold")?
324 {
325 data_config.sr_density_threshold = sr_density_threshold;
326 }
327 data_config.with_sparse_read = with_sparse_read;
328 if let Some(sr_min_gap_size) = config
329 .get_byte_size(b"experimental", b"sparse-read.min-gap-size")?
330 {
331 data_config.sr_min_gap_size = sr_min_gap_size;
332 }
333
334 data_config.with_sparse_read =
335 requirements.contains(SPARSEREVLOG_REQUIREMENT);
336
337 Ok(data_config)
338 }
339 }
340
341 impl Default for RevlogDataConfig {
342 fn default() -> Self {
343 Self {
344 chunk_cache_size: 65536,
345 sr_density_threshold: 0.50,
346 sr_min_gap_size: 262144,
347 try_pending: Default::default(),
348 try_split: Default::default(),
349 check_ambig: Default::default(),
350 mmap_large_index: Default::default(),
351 mmap_index_threshold: Default::default(),
352 uncompressed_cache_factor: Default::default(),
353 uncompressed_cache_count: Default::default(),
354 with_sparse_read: Default::default(),
355 general_delta: Default::default(),
356 }
357 }
358 }
359
360 #[derive(Debug, Clone, Copy, PartialEq)]
361 /// Holds configuration values about how new deltas are computed.
362 ///
363 /// Some attributes are duplicated from [`RevlogDataConfig`] to help having
364 /// each object self contained.
365 pub struct RevlogDeltaConfig {
366 /// Whether deltas can be encoded against arbitrary bases
367 pub general_delta: bool,
368 /// Allow sparse writing of the revlog data
369 pub sparse_revlog: bool,
370 /// Maximum length of a delta chain
371 pub max_chain_len: Option<u64>,
372 /// Maximum distance between a delta chain's start and end
373 pub max_deltachain_span: Option<u64>,
374 /// If `upper_bound_comp` is not None, this is the expected maximal
375 /// gain from compression for the data content
376 pub upper_bound_comp: Option<f64>,
377 /// Should we try a delta against both parents
378 pub delta_both_parents: bool,
379 /// Test delta base candidate groups by chunks of this maximal size
380 pub candidate_group_chunk_size: u64,
381 /// Should we display debug information about delta computation
382 pub debug_delta: bool,
383 /// Trust incoming deltas by default
384 pub lazy_delta: bool,
385 /// Trust the base of incoming deltas by default
386 pub lazy_delta_base: bool,
387 }
388 impl RevlogDeltaConfig {
389 pub fn new(
390 config: &Config,
391 requirements: &HashSet<String>,
392 revlog_type: RevlogType,
393 ) -> Result<Self, HgError> {
394 let mut delta_config = Self {
395 delta_both_parents: config
396 .get_option_no_default(
397 b"storage",
398 b"revlog.optimize-delta-parent-choice",
399 )?
400 .unwrap_or(true),
401 candidate_group_chunk_size: config
402 .get_u64(
403 b"storage",
404 b"revlog.delta-parent-search.candidate-group-chunk-size",
405 )?
406 .unwrap_or_default(),
407 ..Default::default()
408 };
409
410 delta_config.debug_delta =
411 config.get_bool(b"debug", b"revlog.debug-delta")?;
412
413 delta_config.general_delta =
414 requirements.contains(GENERALDELTA_REQUIREMENT);
415
416 let lazy_delta =
417 config.get_bool(b"storage", b"revlog.reuse-external-delta")?;
418
419 if revlog_type == RevlogType::Manifestlog {
420 // upper bound of what we expect from compression
421 // (real life value seems to be 3)
422 delta_config.upper_bound_comp = Some(3.0)
423 }
424
425 let mut lazy_delta_base = false;
426 if lazy_delta {
427 lazy_delta_base = match config.get_option_no_default(
428 b"storage",
429 b"revlog.reuse-external-delta-parent",
430 )? {
431 Some(base) => base,
432 None => config.get_bool(b"format", b"generaldelta")?,
433 };
434 }
435 delta_config.lazy_delta = lazy_delta;
436 delta_config.lazy_delta_base = lazy_delta_base;
437
438 delta_config.max_deltachain_span =
439 match config.get_i64(b"experimental", b"maxdeltachainspan")? {
440 Some(span) => {
441 if span < 0 {
442 None
443 } else {
444 Some(span as u64)
445 }
446 }
447 None => None,
448 };
449
450 delta_config.sparse_revlog =
451 requirements.contains(SPARSEREVLOG_REQUIREMENT);
452
453 delta_config.max_chain_len =
454 config.get_byte_size_no_default(b"format", b"maxchainlen")?;
455
456 Ok(delta_config)
457 }
458 }
459
460 impl Default for RevlogDeltaConfig {
461 fn default() -> Self {
462 Self {
463 delta_both_parents: true,
464 lazy_delta: true,
465 general_delta: Default::default(),
466 sparse_revlog: Default::default(),
467 max_chain_len: Default::default(),
468 max_deltachain_span: Default::default(),
469 upper_bound_comp: Default::default(),
470 candidate_group_chunk_size: Default::default(),
471 debug_delta: Default::default(),
472 lazy_delta_base: Default::default(),
473 }
474 }
475 }
476
477 #[derive(Debug, Default, Clone, Copy, PartialEq)]
478 /// Holds configuration values about the available revlog features
479 pub struct RevlogFeatureConfig {
480 /// The compression engine and its options
481 pub compression_engine: CompressionConfig,
482 /// Can we use censor on this revlog
483 pub censorable: bool,
484 /// Does this revlog use the "side data" feature
485 pub has_side_data: bool,
486 /// Might remove this configuration once the rank computation has no
487 /// impact
488 pub compute_rank: bool,
489 /// Parent order is supposed to be semantically irrelevant, so we
490 /// normally re-sort parents to ensure that the first parent is non-null,
491 /// if there is a non-null parent at all.
492 /// filelog abuses the parent order as a flag to mark some instances of
493 /// meta-encoded files, so allow it to disable this behavior.
494 pub canonical_parent_order: bool,
495 /// Can ellipsis commit be used
496 pub enable_ellipsis: bool,
497 }
498 impl RevlogFeatureConfig {
499 pub fn new(
500 config: &Config,
501 requirements: &HashSet<String>,
502 ) -> Result<Self, HgError> {
503 Ok(Self {
504 compression_engine: CompressionConfig::new(config, requirements)?,
505 enable_ellipsis: requirements.contains(NARROW_REQUIREMENT),
506 ..Default::default()
507 })
508 }
509 }
510
511 /// Read only implementation of revlog.
259 /// Read only implementation of revlog.
512 pub struct Revlog {
260 pub struct Revlog {
513 /// When index and data are not interleaved: bytes of the revlog index.
261 /// When index and data are not interleaved: bytes of the revlog index.
@@ -526,90 +274,6 impl Graph for Revlog {
526 }
274 }
527 }
275 }
528
276
529 #[derive(Debug, Copy, Clone, PartialEq)]
530 pub enum RevlogVersionOptions {
531 V0,
532 V1 { general_delta: bool, inline: bool },
533 V2,
534 ChangelogV2 { compute_rank: bool },
535 }
536
537 /// Options to govern how a revlog should be opened, usually from the
538 /// repository configuration or requirements.
539 #[derive(Debug, Copy, Clone)]
540 pub struct RevlogOpenOptions {
541 /// The revlog version, along with any option specific to this version
542 pub version: RevlogVersionOptions,
543 /// Whether the revlog uses a persistent nodemap.
544 pub use_nodemap: bool,
545 pub delta_config: RevlogDeltaConfig,
546 pub data_config: RevlogDataConfig,
547 pub feature_config: RevlogFeatureConfig,
548 }
549
550 #[cfg(test)]
551 impl Default for RevlogOpenOptions {
552 fn default() -> Self {
553 Self {
554 version: RevlogVersionOptions::V1 {
555 general_delta: true,
556 inline: false,
557 },
558 use_nodemap: true,
559 data_config: Default::default(),
560 delta_config: Default::default(),
561 feature_config: Default::default(),
562 }
563 }
564 }
565
566 impl RevlogOpenOptions {
567 pub fn new(
568 inline: bool,
569 data_config: RevlogDataConfig,
570 delta_config: RevlogDeltaConfig,
571 feature_config: RevlogFeatureConfig,
572 ) -> Self {
573 Self {
574 version: RevlogVersionOptions::V1 {
575 general_delta: data_config.general_delta,
576 inline,
577 },
578 use_nodemap: false,
579 data_config,
580 delta_config,
581 feature_config,
582 }
583 }
584
585 pub fn index_header(&self) -> index::IndexHeader {
586 index::IndexHeader {
587 header_bytes: match self.version {
588 RevlogVersionOptions::V0 => [0, 0, 0, 0],
589 RevlogVersionOptions::V1 {
590 general_delta,
591 inline,
592 } => [
593 0,
594 if general_delta && inline {
595 3
596 } else if general_delta {
597 2
598 } else {
599 u8::from(inline)
600 },
601 0,
602 1,
603 ],
604 RevlogVersionOptions::V2 => 0xDEADu32.to_be_bytes(),
605 RevlogVersionOptions::ChangelogV2 { compute_rank: _ } => {
606 0xD34Du32.to_be_bytes()
607 }
608 },
609 }
610 }
611 }
612
613 impl Revlog {
277 impl Revlog {
614 /// Open a revlog index file.
278 /// Open a revlog index file.
615 ///
279 ///
@@ -19,6 +19,7 use crate::{
19 narrow,
19 narrow,
20 node::NULL_NODE,
20 node::NULL_NODE,
21 operations::{list_rev_tracked_files, ExpandedManifestEntry},
21 operations::{list_rev_tracked_files, ExpandedManifestEntry},
22 options::{default_revlog_options, RevlogOpenOptions},
22 progress::Progress,
23 progress::Progress,
23 repo::Repo,
24 repo::Repo,
24 sparse,
25 sparse,
@@ -28,7 +29,7 use crate::{
28 path_auditor::PathAuditor,
29 path_auditor::PathAuditor,
29 },
30 },
30 vfs::{is_on_nfs_mount, VfsImpl},
31 vfs::{is_on_nfs_mount, VfsImpl},
31 DirstateParents, RevlogError, RevlogOpenOptions, UncheckedRevision,
32 DirstateParents, RevlogError, UncheckedRevision,
32 };
33 };
33 use crossbeam_channel::{Receiver, Sender};
34 use crossbeam_channel::{Receiver, Sender};
34 use rayon::prelude::*;
35 use rayon::prelude::*;
@@ -89,7 +90,11 pub fn update_from_null(
89 return Ok(0);
90 return Ok(0);
90 }
91 }
91 let store_vfs = &repo.store_vfs();
92 let store_vfs = &repo.store_vfs();
92 let options = repo.default_revlog_options(crate::RevlogType::Filelog)?;
93 let options = default_revlog_options(
94 repo.config(),
95 repo.requirements(),
96 crate::RevlogType::Filelog,
97 )?;
93 let (errors_sender, errors_receiver) = crossbeam_channel::unbounded();
98 let (errors_sender, errors_receiver) = crossbeam_channel::unbounded();
94 let (files_sender, files_receiver) = crossbeam_channel::unbounded();
99 let (files_sender, files_receiver) = crossbeam_channel::unbounded();
95 let working_directory_path = &repo.working_directory_path();
100 let working_directory_path = &repo.working_directory_path();
@@ -23,16 +23,17 use hg::lock::LockError;
23 use hg::manifest::Manifest;
23 use hg::manifest::Manifest;
24 use hg::matchers::{AlwaysMatcher, IntersectionMatcher};
24 use hg::matchers::{AlwaysMatcher, IntersectionMatcher};
25 use hg::repo::Repo;
25 use hg::repo::Repo;
26 use hg::revlog::options::{default_revlog_options, RevlogOpenOptions};
26 use hg::utils::debug::debug_wait_for_file;
27 use hg::utils::debug::debug_wait_for_file;
27 use hg::utils::files::{
28 use hg::utils::files::{
28 get_bytes_from_os_str, get_bytes_from_os_string, get_path_from_bytes,
29 get_bytes_from_os_str, get_bytes_from_os_string, get_path_from_bytes,
29 };
30 };
30 use hg::utils::hg_path::{hg_path_to_path_buf, HgPath};
31 use hg::utils::hg_path::{hg_path_to_path_buf, HgPath};
32 use hg::DirstateStatus;
31 use hg::Revision;
33 use hg::Revision;
32 use hg::StatusError;
34 use hg::StatusError;
33 use hg::StatusOptions;
35 use hg::StatusOptions;
34 use hg::{self, narrow, sparse};
36 use hg::{self, narrow, sparse};
35 use hg::{DirstateStatus, RevlogOpenOptions};
36 use hg::{PatternFileWarning, RevlogType};
37 use hg::{PatternFileWarning, RevlogType};
37 use log::info;
38 use log::info;
38 use rayon::prelude::*;
39 use rayon::prelude::*;
@@ -383,8 +384,11 pub fn run(invocation: &crate::CliInvoca
383 })?;
384 })?;
384 let working_directory_vfs = repo.working_directory_vfs();
385 let working_directory_vfs = repo.working_directory_vfs();
385 let store_vfs = repo.store_vfs();
386 let store_vfs = repo.store_vfs();
386 let revlog_open_options =
387 let revlog_open_options = default_revlog_options(
387 repo.default_revlog_options(RevlogType::Manifestlog)?;
388 repo.config(),
389 repo.requirements(),
390 RevlogType::Manifestlog,
391 )?;
388 let res: Vec<_> = take(&mut ds_status.unsure)
392 let res: Vec<_> = take(&mut ds_status.unsure)
389 .into_par_iter()
393 .into_par_iter()
390 .map(|to_check| {
394 .map(|to_check| {
General Comments 0
You need to be logged in to leave comments. Login now