branching: merge stable into default

doc/gendoc.py

@@ -21,7 +21,7 b' except ImportError:'
 # available. Relax C module requirements.
 os.environ['HGMODULEPOLICY'] = 'allow'
 # import from the live mercurial repo
-sys.path.insert(0, "..")
+sys.path.insert(0, os.path.abspath(".."))
 from mercurial import demandimport
 
 demandimport.enable()

mercurial/debugcommands.py

@@ -46,6 +46,7 b' from . import ('
     context,
     copies,
     dagparser,
+    dirstateutils,
     encoding,
     error,
     exchange,
@@ -939,6 +940,12 b' def debugdeltachain(ui, repo, file_=None'
         (b'', b'datesort', None, _(b'sort by saved mtime')),
         (
             b'',
+            b'docket',
+            False,
+            _(b'display the docket (metadata file) instead'),
+        ),
+        (
+            b'',
             b'all',
             False,
             _(b'display dirstate-v2 tree nodes that would not exist in v1'),
@@ -949,6 +956,33 b' def debugdeltachain(ui, repo, file_=None'
 def debugstate(ui, repo, **opts):
     """show the contents of the current dirstate"""
 
+    if opts.get("docket"):
+        if not repo.dirstate._use_dirstate_v2:
+            raise error.Abort(_(b'dirstate v1 does not have a docket'))
+
+        docket = repo.dirstate._map.docket
+        (
+            start_offset,
+            root_nodes,
+            nodes_with_entry,
+            nodes_with_copy,
+            unused_bytes,
+            _unused,
+            ignore_pattern,
+        ) = dirstateutils.v2.TREE_METADATA.unpack(docket.tree_metadata)
+
+        ui.write(_(b"size of dirstate data: %d\n") % docket.data_size)
+        ui.write(_(b"data file uuid: %s\n") % docket.uuid)
+        ui.write(_(b"start offset of root nodes: %d\n") % start_offset)
+        ui.write(_(b"number of root nodes: %d\n") % root_nodes)
+        ui.write(_(b"nodes with entries: %d\n") % nodes_with_entry)
+        ui.write(_(b"nodes with copies: %d\n") % nodes_with_copy)
+        ui.write(_(b"number of unused bytes: %d\n") % unused_bytes)
+        ui.write(
+            _(b"ignore pattern hash: %s\n") % binascii.hexlify(ignore_pattern)
+        )
+        return
+
     nodates = not opts['dates']
     if opts.get('nodates') is not None:
         nodates = True
@@ -983,22 +1017,6 b' def debugstate(ui, repo, **opts):'
 
 
 @command(
-    b'debugdirstateignorepatternshash',
-    [],
-    _(b''),
-)
-def debugdirstateignorepatternshash(ui, repo, **opts):
-    """show the hash of ignore patterns stored in dirstate if v2,
-    or nothing for dirstate-v2
-    """
-    if repo.dirstate._use_dirstate_v2:
-        docket = repo.dirstate._map.docket
-        hash_len = 20  # 160 bits for SHA-1
-        hash_bytes = docket.tree_metadata[-hash_len:]
-        ui.write(binascii.hexlify(hash_bytes) + b'\n')
-
-
-@command(
     b'debugdiscovery',
     [
         (b'', b'old', None, _(b'use old-style discovery')),

mercurial/helptext/rust.txt

@@ -28,7 +28,8 b' in progress. For more experimental work '
 Checking for Rust
 =================
 
-You may already have the Rust extensions depending on how you install Mercurial.
+You may already have the Rust extensions depending on how you install
+Mercurial::
 
   $ hg debuginstall | grep -i rust
   checking Rust extensions (installed)
@@ -46,7 +47,7 b' version to use.'
 Using pip
 ---------
 
-Users of `pip` can install the Rust extensions with the following command:
+Users of `pip` can install the Rust extensions with the following command::
 
   $ pip install mercurial --global-option --rust --no-use-pep517
 

mercurial/localrepo.py

@@ -3175,7 +3175,7 b' class localrepository:'
             # Save commit message in case this transaction gets rolled back
             # (e.g. by a pretxncommit hook).  Leave the content alone on
             # the assumption that the user will use the same editor again.
-            msgfn = self.savecommitmessage(cctx._text)
+            msg_path = self.savecommitmessage(cctx._text)
 
             # commit subs and write new state
             if subs:
@@ -3205,13 +3205,14 b' class localrepository:'
             except:  # re-raises
                 if edited:
                     self.ui.write(
-                        _(b'note: commit message saved in %s\n') % msgfn
+                        _(b'note: commit message saved in %s\n') % msg_path
                     )
                     self.ui.write(
                         _(
                             b"note: use 'hg commit --logfile "
-                            b".hg/last-message.txt --edit' to reuse it\n"
+                            b"%s --edit' to reuse it\n"
                         )
+                        % msg_path
                     )
                 raise
 

relnotes/6.1

@@ -1,5 +1,23 b''
 '''This is the last release to support Python 2. Mercurial is Python 3 only starting with 6.2'''
 
+= Mercurial 6.1.2 =
+
+ * Improve Windows test suite
+ * Fix `debuglock` not ignoring a missing lockfile when forcing a lock
+ * Improve help of `ui.large-file-limit`
+ * Set the large-file-limit to 10MB (from 10MiB) for clarity
+ * While rewriting desc hashes, ignore ambiguous prefix "hashes"
+ * Fix a crash in partial amend with copies
+ * Fix a py3 compatiblity bug
+ * Fix incorrect metadata causing dirstate-v2 data loss in edge case
+ * Fix cleanup of old dirstate-v2 data files when using `rhg`
+ * Make reference to `.hg/last_message.txt` relative in commit
+ * Fix an infinite hang when `rhg` is used in the background
+ * Fix Python DLL loading bug in Windows
+ * Add `--docket` flag to `debugstate` to check out dirstate-v2 metadata
+ * Remove `debugdirstateignorepatternhash` in favor of `debugstate --docket`
+ * Fix incorrect metadata causing systematic complete dirstate-v2 rewrite
+
 = Mercurial 6.1.1 =
 
  * Fix Rust compilation on `aarcch64`

rust/Cargo.lock

@@ -590,9 +590,9 b' checksum = "e2abad23fbc42b3700f2f279844d'
 
 [[package]]
 name = "libc"
-version = "0.2.119"
+version = "0.2.124"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
+checksum = "21a41fed9d98f27ab1c6d161da622a4fa35e8a54a8adc24bbf3ddd0ef70b0e50"
 
 [[package]]
 name = "libm"
@@ -1032,6 +1032,7 b' dependencies = ['
  "micro-timer 0.4.0",
  "regex",
  "users",
+ "which",
 ]
 
 [[package]]
@@ -1251,6 +1252,17 b' source = "registry+https://github.com/ru'
 checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
 
 [[package]]
+name = "which"
+version = "4.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
+dependencies = [
+ "either",
+ "lazy_static",
+ "libc",
+]
+
+[[package]]
 name = "winapi"
 version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"

rust/hg-core/src/dirstate_tree/on_disk.rs

@@ -622,13 +622,18 b' pub(super) fn write('
 
     let root_nodes = writer.write_nodes(dirstate_map.root.as_ref())?;
 
+    let unreachable_bytes = if append {
+        dirstate_map.unreachable_bytes
+    } else {
+        0
+    };
     let meta = TreeMetadata {
         root_nodes,
         nodes_with_entry_count: dirstate_map.nodes_with_entry_count.into(),
         nodes_with_copy_source_count: dirstate_map
             .nodes_with_copy_source_count
             .into(),
-        unreachable_bytes: dirstate_map.unreachable_bytes.into(),
+        unreachable_bytes: unreachable_bytes.into(),
         unused: [0; 4],
         ignore_patterns_hash: dirstate_map.ignore_patterns_hash,
     };

rust/hg-core/src/exit_codes.rs

@@ -17,3 +17,6 b' pub const UNSUCCESSFUL: ExitCode = 1;'
 
 /// Command or feature not implemented by rhg
 pub const UNIMPLEMENTED: ExitCode = 252;
+
+/// The fallback path is not valid
+pub const INVALID_FALLBACK: ExitCode = 253;

rust/hg-core/src/repo.rs

@@ -424,25 +424,32 b' impl Repo {'
         // it’s unset
         let parents = self.dirstate_parents()?;
         let (packed_dirstate, old_uuid_to_remove) = if self.has_dirstate_v2() {
-            let uuid = self.dirstate_data_file_uuid.get_or_init(self)?;
-            let mut uuid = uuid.as_ref();
-            let can_append = uuid.is_some();
+            let uuid_opt = self.dirstate_data_file_uuid.get_or_init(self)?;
+            let uuid_opt = uuid_opt.as_ref();
+            let can_append = uuid_opt.is_some();
             let (data, tree_metadata, append, old_data_size) =
                 map.pack_v2(can_append)?;
-            if !append {
-                uuid = None
-            }
-            let (uuid, old_uuid) = if let Some(uuid) = uuid {
-                let as_str = std::str::from_utf8(uuid)
-                    .map_err(|_| {
-                        HgError::corrupted("non-UTF-8 dirstate data file ID")
-                    })?
-                    .to_owned();
-                let old_uuid_to_remove = Some(as_str.to_owned());
-                (as_str, old_uuid_to_remove)
-            } else {
-                (DirstateDocket::new_uid(), None)
+
+            // Reuse the uuid, or generate a new one, keeping the old for
+            // deletion.
+            let (uuid, old_uuid) = match uuid_opt {
+                Some(uuid) => {
+                    let as_str = std::str::from_utf8(uuid)
+                        .map_err(|_| {
+                            HgError::corrupted(
+                                "non-UTF-8 dirstate data file ID",
+                            )
+                        })?
+                        .to_owned();
+                    if append {
+                        (as_str, None)
+                    } else {
+                        (DirstateDocket::new_uid(), Some(as_str))
+                    }
+                }
+                None => (DirstateDocket::new_uid(), None),
             };
+
             let data_filename = format!("dirstate.{}", uuid);
             let data_filename = self.hg_vfs().join(data_filename);
             let mut options = std::fs::OpenOptions::new();

rust/rhg/Cargo.toml

@@ -21,3 +21,4 b' regex = "1.5.5"'
 env_logger = "0.9.0"
 format-bytes = "0.3.0"
 users = "0.11.0"
+which = "4.2.5"

rust/rhg/src/error.rs

@@ -29,6 +29,9 b' pub enum CommandError {'
     /// `rhg` may attempt to silently fall back to Python-based `hg`, which
     /// may or may not support this feature.
     UnsupportedFeature { message: Vec<u8> },
+    /// The fallback executable does not exist (or has some other problem if
+    /// we end up being more precise about broken fallbacks).
+    InvalidFallback { path: Vec<u8>, err: String },
 }
 
 impl CommandError {

rust/rhg/src/main.rs

@@ -13,6 +13,7 b' use hg::utils::files::{get_bytes_from_os'
 use hg::utils::SliceExt;
 use std::collections::HashSet;
 use std::ffi::OsString;
+use std::os::unix::prelude::CommandExt;
 use std::path::PathBuf;
 use std::process::Command;
 
@@ -381,12 +382,14 b' fn exit_code('
             }
         }
         Err(CommandError::Unsuccessful) => exit_codes::UNSUCCESSFUL,
-
         // Exit with a specific code and no error message to let a potential
         // wrapper script fallback to Python-based Mercurial.
         Err(CommandError::UnsupportedFeature { .. }) => {
             exit_codes::UNIMPLEMENTED
         }
+        Err(CommandError::InvalidFallback { .. }) => {
+            exit_codes::INVALID_FALLBACK
+        }
     }
 }
 
@@ -432,6 +435,17 b" fn exit<'a>("
         } else {
             log::debug!("falling back (see trace-level log)");
             log::trace!("{}", local_to_utf8(message));
+            if let Err(err) = which::which(executable_path) {
+                exit_no_fallback(
+                    ui,
+                    OnUnsupported::Abort,
+                    Err(CommandError::InvalidFallback {
+                        path: executable.to_owned(),
+                        err: err.to_string(),
+                    }),
+                    use_detailed_exit_code,
+                )
+            }
             // `args` is now `argv[1..]` since we’ve already consumed
             // `argv[0]`
             let mut command = Command::new(executable_path);
@@ -439,19 +453,19 b" fn exit<'a>("
             if let Some(initial) = initial_current_dir {
                 command.current_dir(initial);
             }
-            let result = command.status();
-            match result {
-                Ok(status) => std::process::exit(
-                    status.code().unwrap_or(exit_codes::ABORT),
-                ),
-                Err(error) => {
-                    let _ = ui.write_stderr(&format_bytes!(
-                        b"tried to fall back to a '{}' sub-process but got error {}\n",
-                        executable, format_bytes::Utf8(error)
-                    ));
-                    on_unsupported = OnUnsupported::Abort
-                }
-            }
+            // We don't use subprocess because proper signal handling is harder
+            // and we don't want to keep `rhg` around after a fallback anyway.
+            // For example, if `rhg` is run in the background and falls back to
+            // `hg` which, in turn, waits for a signal, we'll get stuck if
+            // we're doing plain subprocess.
+            //
+            // If `exec` returns, we can only assume our process is very broken
+            // (see its documentation), so only try to forward the error code
+            // when exiting.
+            let err = command.exec();
+            std::process::exit(
+                err.raw_os_error().unwrap_or(exit_codes::ABORT),
+            );
         }
     }
     exit_no_fallback(ui, on_unsupported, result, use_detailed_exit_code)
@@ -488,6 +502,13 b' fn exit_no_fallback('
                 OnUnsupported::Fallback { .. } => unreachable!(),
             }
         }
+        Err(CommandError::InvalidFallback { path, err }) => {
+            let _ = ui.write_stderr(&format_bytes!(
+                b"abort: invalid fallback '{}': {}\n",
+                path,
+                err.as_bytes(),
+            ));
+        }
     }
     std::process::exit(exit_code(&result, use_detailed_exit_code))
 }

tests/test-completion.t

@@ -94,7 +94,6 b' Show debug commands if there are no othe'
   debugdate
   debugdeltachain
   debugdirstate
-  debugdirstateignorepatternshash
   debugdiscovery
   debugdownload
   debugextensions
@@ -285,8 +284,7 b' Show all commands + options'
   debugdata: changelog, manifest, dir
   debugdate: extended
   debugdeltachain: changelog, manifest, dir, template
-  debugdirstateignorepatternshash: 
-  debugdirstate: nodates, dates, datesort, all
+  debugdirstate: nodates, dates, datesort, docket, all
   debugdiscovery: old, nonheads, rev, seed, local-as-revs, remote-as-revs, ssh, remotecmd, insecure, template
   debugdownload: output
   debugextensions: template

tests/test-dirstate.t

@@ -119,4 +119,88 b' infinite loop.'
   C hgext3rd/__init__.py
 
   $ cd ..
+
+Check that the old dirstate data file is removed correctly and the new one is
+valid.
+
+  $ dirstate_data_files () {
+  >   find .hg -maxdepth 1 -name "dirstate.*"
+  > }
+
+  $ find_dirstate_uuid () {
+  >   hg debugstate --docket | grep uuid | sed 's/.*uuid: \(.*\)/\1/'
+  > }
+
+  $ dirstate_uuid_has_not_changed () {
+  >   # Non-Rust always rewrites the whole dirstate
+  >   if [ $# -eq 1 ] || ([ -n "$HGMODULEPOLICY" ] && [ -z "${HGMODULEPOLICY##*rust*}" ]) || [ -n "$RHG_INSTALLED_AS_HG" ]; then
+  >     test $current_uid = $(find_dirstate_uuid)
+  >   else
+  >     echo "not testing because using Python implementation"
+  >   fi
+  > }
+
+  $ cd ..
+  $ hg init append-mostly
+  $ cd append-mostly
+  $ mkdir dir dir2
+  $ touch dir/a dir/b dir/c dir/d dir/e dir2/f
+  $ hg commit -Aqm initial
+  $ hg st
+  $ dirstate_data_files | wc -l
+   *1 (re)
+  $ current_uid=$(find_dirstate_uuid)
+
+Nothing changes here
+
+  $ hg st
+  $ dirstate_data_files | wc -l
+   *1 (re)
+  $ dirstate_uuid_has_not_changed
+  not testing because using Python implementation (no-rust no-rhg !)
+
+Trigger an append with a small change
+
+  $ echo "modified" > dir2/f
+  $ hg st
+  M dir2/f
+  $ dirstate_data_files | wc -l
+   *1 (re)
+  $ dirstate_uuid_has_not_changed
+  not testing because using Python implementation (no-rust no-rhg !)
+
+Unused bytes counter is non-0 when appending
+  $ touch file
+  $ hg add file
+  $ current_uid=$(find_dirstate_uuid)
+
+Trigger a rust/rhg run which updates the unused bytes value
+  $ hg st
+  M dir2/f
+  A file
+  $ dirstate_data_files | wc -l
+   *1 (re)
+  $ dirstate_uuid_has_not_changed
+  not testing because using Python implementation (no-rust no-rhg !)
+
+  $ hg debugstate --docket | grep unused
+  number of unused bytes: 0 (no-rust no-rhg !)
+  number of unused bytes: [1-9]\d* (re) (rhg no-rust !)
+  number of unused bytes: [1-9]\d* (re) (rust no-rhg !)
+  number of unused bytes: [1-9]\d* (re) (rust rhg !)
+
+Delete most of the dirstate to trigger a non-append
+  $ hg rm dir/a dir/b dir/c dir/d
+  $ dirstate_data_files | wc -l
+   *1 (re)
+  $ dirstate_uuid_has_not_changed also-if-python
+  [1]
+
+Check that unused bytes counter is reset when creating a new docket
+
+  $ hg debugstate --docket | grep unused
+  number of unused bytes: 0
+
 #endif
+
+  $ cd ..

tests/test-help.t

@@ -1013,8 +1013,6 b' Test list of internal help commands'
                  dump information about delta chains in a revlog
    debugdirstate
                  show the contents of the current dirstate
-   debugdirstateignorepatternshash
-                 show the hash of ignore patterns stored in dirstate if v2,
    debugdiscovery
                  runs the changeset discovery protocol in isolation
    debugdownload

tests/test-hgignore.t

@@ -418,14 +418,14 b' This is an optimization that is only rel'
   $ hg status > /dev/null
   $ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1
   sha1=6e315b60f15fb5dfa02be00f3e2c8f923051f5ff
-  $ hg debugdirstateignorepatternshash
-  6e315b60f15fb5dfa02be00f3e2c8f923051f5ff
+  $ hg debugstate --docket | grep ignore
+  ignore pattern hash: 6e315b60f15fb5dfa02be00f3e2c8f923051f5ff
 
   $ echo rel > .hg/testhgignorerel
   $ hg status > /dev/null
   $ cat .hg/testhgignore .hg/testhgignorerel .hgignore dir2/.hgignore dir1/.hgignore dir1/.hgignoretwo | $TESTDIR/f --sha1
   sha1=dea19cc7119213f24b6b582a4bae7b0cb063e34e
-  $ hg debugdirstateignorepatternshash
-  dea19cc7119213f24b6b582a4bae7b0cb063e34e
+  $ hg debugstate --docket | grep ignore
+  ignore pattern hash: dea19cc7119213f24b6b582a4bae7b0cb063e34e
 
 #endif

tests/test-histedit-edit.t

@@ -356,6 +356,8 b' check saving last-message.txt, at first'
   A f
 
   $ rm -f .hg/last-message.txt
+  $ mkdir dir
+  $ cd dir
   $ HGEDITOR="sh $TESTTMP/editor.sh" hg histedit tip --commands - 2>&1 << EOF
   > mess 1fd3b2fe7754 f
   > EOF
@@ -372,10 +374,11 b' check saving last-message.txt, at first'
   ====
   transaction abort!
   rollback completed
-  note: commit message saved in .hg/last-message.txt
-  note: use 'hg commit --logfile .hg/last-message.txt --edit' to reuse it
+  note: commit message saved in ../.hg/last-message.txt
+  note: use 'hg commit --logfile ../.hg/last-message.txt --edit' to reuse it
   abort: pretxncommit.unexpectedabort hook exited with status 1
   [40]
+  $ cd ..
   $ cat .hg/last-message.txt
   f
   

tests/test-rhg.t

@@ -179,15 +179,8 b' Fallback to Python'
   [1]
 
   $ rhg cat original --exclude="*.rs" --config rhg.fallback-executable=hg-non-existent
-  tried to fall back to a 'hg-non-existent' sub-process but got error $ENOENT$
-  unsupported feature: error: Found argument '--exclude' which wasn't expected, or isn't valid in this context
-  
-  USAGE:
-      rhg cat [OPTIONS] <FILE>...
-  
-  For more information try --help
-  
-  [252]
+  abort: invalid fallback 'hg-non-existent': cannot find binary path
+  [253]
 
   $ rhg cat original --exclude="*.rs" --config rhg.fallback-executable=rhg
   Blocking recursive fallback. The 'rhg.fallback-executable = rhg' config points to `rhg` itself.