branching: merge stable into default

.hgsigs

@@ -240,3 +240,4 b' c890d8b8bc59b18e5febf60caada629df5356ee2'
 59466b13a3ae0e29a5d4f485393e516cfbb057d0 0 iQHNBAABCgA3FiEEH2b4zfZU6QXBHaBhoR4BzQ4F2VYFAmO1XgoZHGFscGhhcmVAcmFwaGFlbGdvbWVzLmRldgAKCRChHgHNDgXZVn8nDACU04KbPloLl+if6DQYreESnF9LU8C+qnLC/j5RRuaFNh/ec6C3DzLWqWdmnWA/siV3nUR1bXHfTui95azxJfYvWoXH2R2yam+YhE256B4rDDYWS1LI9kNNM+A33xcPS2HxVowkByhjB5FPKR6I90dX42BYJpTS5s/VPx63wXLznjFWuD7XJ3P0VI7D72j/+6EQCmHaAUEE5bO00Ob2JxmzJlaP+02fYc814PAONE2/ocfR0aExAVS3VA+SJGXnXTVpoaHr7NJKC2sBLFsdnhIRwtCf3rtGEvIJ5v2U2xx0ZEz/mimtGzW5ovkthobV4mojk0DRz7xBtA96pOGSRTD8QndIsdMCUipo8zZ/AGAMByCtsQOX7OYhR6gp+I6+iPh8fTR5oCbkO7cizDDQtXcrR5OT/BDH9xkAF1ghNL8o23a09/wfZ9NPg5zrh/4T/dFfoe2COlkAJJ1ttDPYyQkCfMsoWm3OXk6xJ3ExVbwkZzUDQSzsxGS+oxbFDWJZ64Q=
 8830004967ad865ead89c28a410405a6e71e0796 0 iQHNBAABCgA3FiEEH2b4zfZU6QXBHaBhoR4BzQ4F2VYFAmQAsOQZHGFscGhhcmVAcmFwaGFlbGdvbWVzLmRldgAKCRChHgHNDgXZVl7XC/0W+Wd4gzMUbaot+NVIZTpubNw3KHBDXrlMgwQgCDg7qcqJnVuT1NNEy5sRELjZOO0867k+pBchZaxdmAiFwY1W76+7nwiLBqfCkYgYY0iQe48JHTq9kCgohvx9PSEVbUsScmqAQImd5KzErjhsLj8D2FiFIrcMyqsCBq4ZPs0Ey7lVKu6q3z5eDjlrxUIr0up6yKvgBxhY0GxyTp6DGoinzlFMEadiJlsvlwO4C6UpzKiCGMeKNT5xHK/Hx3ChrOH2Yuu1fHaPLJ+ZpXjR33ileVYlkQrh1D6fWHXcP7ZuwsEKREtgsw1YjYczGFwmhBO362bNi5wy33mBtCvcIAqpsI0rMrExs66qqbfyG+Yp1dvkgzUfdhbYFHA+mvg3/YTSD9dLKzzsb69LM87+dvcLqhBJ0nEAuBmAzU5ECkoArbiwMT96NhhjLPRmJJdHNo0IDos/LBGTgkOZ6iqIx8Xm/tgjBjFJG8B+IVy3laNgun4AZ9Ejc3ahIfhJUIo2j8o=
 05de4896508e8ec387b33eb30d8aab78d1c8e9e4 0 iQHNBAABCgA3FiEEH2b4zfZU6QXBHaBhoR4BzQ4F2VYFAmQBI2AZHGFscGhhcmVAcmFwaGFlbGdvbWVzLmRldgAKCRChHgHNDgXZVrRZC/wJyPOJoxpjEJZaRoBmWtkOlf0Y0TyEb6wd8tZIVALNDYZMSMqT7UBjFmaZijOYndUW7ZCj1hKShaIw80vY/hjJ3KZMODY9t91SOwmrVaGrCUeF1tXkuhEgwxfkekPWLxYYc688gLb6oc3FBm//lucNGrOWBXw6yhm1dUcndHXXpafjJslKAHwJN7vI5q69SxvS6SlJUzh/RFWYLnbZ2Qi35ixkU12FZiYVzxDl2i7XbhVoT5mit6VTU7Wh4BMSYuorAv937sF9Y6asE7sQUYHC2C2qjp8S5uFXV/IrhCPbJyWVc4ymPm58Eh6SmItC9zHDviFF9aFoZMK/lfK3Dqumu3T9x6ZYcxulpjNsM0/yv9OiiWbw33PnNb74A9uwrxZHB3XexXiigBUlUzO4lJQ5Oe1rhpPfPPRVyxaeZ8/cPmoJjCuwoiG0YtUeNH5PkHi05O0/hLR9PftDY8oMyzOBErSqjMjZ6OTkFFgk3dI9rHU72C1KL9Jh5uHwEQchBmg=
+f14864fffdcab725d9eac6d4f4c07be05a35f59a 0 iQHNBAABCgA3FiEEH2b4zfZU6QXBHaBhoR4BzQ4F2VYFAmQc3KUZHGFscGhhcmVAcmFwaGFlbGdvbWVzLmRldgAKCRChHgHNDgXZVnYZDACh1Bcj8Yu3t8pO22SKWJnz8Ndw9Hvw+ifLaRxFUxKtqUYvy3CIl2qt8k7V13M25qw0061SKgcvNdjtkOhdmtFHNAbqryy0nK9oSZ2GfndmJfMxm9ixF/CcHrx+MmsklEz2woApViHW5PrmgKvZNsStQ5NM457Yx3B4nsT9b8t03NzdNiZRM+RZOkZ+4OdSbiB6hYuTqEFIi2YM+gfVM5Z7H8sEFBkUCtuwUjFGaWThZGGhAcqD5E7p/Lkjv4e4tzyHOzHDgdd+OCAkcbib6/E3Q1MlQ1x7CKpJ190T8R35CzAIMBVoTSI+Ov7OKw1OfGdeCvMVJsKUvqY3zrPawmJB6pG7GoVPEu5pU65H51U3Plq3GhsekUrKWY/BSHV9FOqpKZdnxOAllfWcjLYpbC/fM3l8uuQVcPAs89GvWKnDuE/NWCDYzDAYE++s/H4tP3Chv6yQbPSv/lbccst7OfLLDtXgRHIyEWLo392X3mWzhrkNtfJkBdi39uH9Aoh7pN0=

.hgtags

@@ -256,3 +256,4 b' c890d8b8bc59b18e5febf60caada629df5356ee2'
 59466b13a3ae0e29a5d4f485393e516cfbb057d0 6.3.2
 8830004967ad865ead89c28a410405a6e71e0796 6.3.3
 05de4896508e8ec387b33eb30d8aab78d1c8e9e4 6.4rc0
+f14864fffdcab725d9eac6d4f4c07be05a35f59a 6.4

mercurial/changelog.py

@@ -481,7 +481,7 b' class changelog(revlog.revlog):'
             self._delaybuf = None
         self._divert = False
         # split when we're done
-        self._enforceinlinesize(tr)
+        self._enforceinlinesize(tr, side_write=False)
 
     def _writepending(self, tr):
         """create a file containing the unfinalized state for
@@ -512,9 +512,9 b' class changelog(revlog.revlog):'
 
         return False
 
-    def _enforceinlinesize(self, tr):
+    def _enforceinlinesize(self, tr, side_write=True):
         if not self._delayed:
-            revlog.revlog._enforceinlinesize(self, tr)
+            revlog.revlog._enforceinlinesize(self, tr, side_write=side_write)
 
     def read(self, nodeorrev):
         """Obtain data from a parsed changelog revision.

mercurial/debugcommands.py

@@ -803,11 +803,12 b' def debugdeltachain(ui, repo, file_=None'
     # security to avoid crash on corrupted revlogs
     total_revs = len(index)
 
+    chain_size_cache = {}
+
     def revinfo(rev):
         e = index[rev]
         compsize = e[revlog_constants.ENTRY_DATA_COMPRESSED_LENGTH]
         uncompsize = e[revlog_constants.ENTRY_DATA_UNCOMPRESSED_LENGTH]
-        chainsize = 0
 
         base = e[revlog_constants.ENTRY_DELTA_BASE]
         p1 = e[revlog_constants.ENTRY_PARENT_1]
@@ -870,11 +871,17 b' def debugdeltachain(ui, repo, file_=None'
                 deltatype = b'prev'
 
         chain = r._deltachain(rev)[0]
-        for iterrev in chain:
-            e = index[iterrev]
-            chainsize += e[revlog_constants.ENTRY_DATA_COMPRESSED_LENGTH]
-
-        return p1, p2, compsize, uncompsize, deltatype, chain, chainsize
+        chain_size = 0
+        for iter_rev in reversed(chain):
+            cached = chain_size_cache.get(iter_rev)
+            if cached is not None:
+                chain_size += cached
+                break
+            e = index[iter_rev]
+            chain_size += e[revlog_constants.ENTRY_DATA_COMPRESSED_LENGTH]
+        chain_size_cache[rev] = chain_size
+
+        return p1, p2, compsize, uncompsize, deltatype, chain, chain_size
 
     fm = ui.formatter(b'debugdeltachain', opts)
 

mercurial/filelog.py

@@ -25,7 +25,7 b' from .revlogutils import ('
 
 @interfaceutil.implementer(repository.ifilestorage)
 class filelog:
-    def __init__(self, opener, path):
+    def __init__(self, opener, path, try_split=False):
         self._revlog = revlog.revlog(
             opener,
             # XXX should use the unencoded path
@@ -33,6 +33,7 b' class filelog:'
             radix=b'/'.join((b'data', path)),
             censorable=True,
             canonical_parent_order=False,  # see comment in revlog.py
+            try_split=try_split,
         )
         # Full name of the user visible file, relative to the repository root.
         # Used by LFS.
@@ -256,8 +257,8 b' class filelog:'
 class narrowfilelog(filelog):
     """Filelog variation to be used with narrow stores."""
 
-    def __init__(self, opener, path, narrowmatch):
-        super(narrowfilelog, self).__init__(opener, path)
+    def __init__(self, opener, path, narrowmatch, try_split=False):
+        super(narrowfilelog, self).__init__(opener, path, try_split=try_split)
         self._narrowmatch = narrowmatch
 
     def renamed(self, node):

mercurial/hook.py

@@ -191,6 +191,11 b' def _exthook(ui, repo, htype, name, cmd,'
         cwd = encoding.getcwd()
     r = ui.system(cmd, environ=env, cwd=cwd, blockedtag=b'exthook-%s' % (name,))
 
+    if repo is not None and repo.currentwlock() is None:
+        repo.invalidatedirstate()
+    if repo is not None and repo.currentlock() is None:
+        repo.invalidate()
+
     duration = util.timer() - starttime
     ui.log(
         b'exthook',

mercurial/interfaces/repository.py

@@ -1810,6 +1810,9 b' class ilocalrepositorymain(interfaceutil'
     def lock(wait=True):
         """Lock the repository store and return a lock instance."""
 
+    def currentlock():
+        """Return the lock if it's held or None."""
+
     def wlock(wait=True):
         """Lock the non-store parts of the repository."""
 

mercurial/localrepo.py

@@ -1240,7 +1240,12 b' class revlogfilestorage:'
         if path.startswith(b'/'):
             path = path[1:]
 
-        return filelog.filelog(self.svfs, path)
+        try_split = (
+            self.currenttransaction() is not None
+            or txnutil.mayhavepending(self.root)
+        )
+
+        return filelog.filelog(self.svfs, path, try_split=try_split)
 
 
 @interfaceutil.implementer(repository.ilocalrepositoryfilestorage)
@@ -1251,7 +1256,13 b' class revlognarrowfilestorage:'
         if path.startswith(b'/'):
             path = path[1:]
 
-        return filelog.narrowfilelog(self.svfs, path, self._storenarrowmatch)
+        try_split = (
+            self.currenttransaction() is not None
+            or txnutil.mayhavepending(self.root)
+        )
+        return filelog.narrowfilelog(
+            self.svfs, path, self._storenarrowmatch, try_split=try_split
+        )
 
 
 def makefilestorage(requirements, features, **kwargs):
@@ -1794,10 +1805,29 b' class localrepository:'
         )
 
     def _dirstatevalidate(self, node):
+        okay = True
         try:
             self.changelog.rev(node)
+        except error.LookupError:
+            # If the parent are unknown it might just be because the changelog
+            # in memory is lagging behind the dirstate in memory. So try to
+            # refresh the changelog first.
+            #
+            # We only do so if we don't hold the lock, if we do hold the lock
+            # the invalidation at that time should have taken care of this and
+            # something is very fishy.
+            if self.currentlock() is None:
+                self.invalidate()
+                try:
+                    self.changelog.rev(node)
+                except error.LookupError:
+                    okay = False
+            else:
+                # XXX we should consider raising an error here.
+                okay = False
+        if okay:
             return node
-        except error.LookupError:
+        else:
             if not self._dirstatevalidatewarned:
                 self._dirstatevalidatewarned = True
                 self.ui.warn(
@@ -3130,6 +3160,10 b' class localrepository:'
         """Returns the wlock if it's held, or None if it's not."""
         return self._currentlock(self._wlockref)
 
+    def currentlock(self):
+        """Returns the lock if it's held, or None if it's not."""
+        return self._currentlock(self._lockref)
+
     def checkcommitpatterns(self, wctx, match, status, fail):
         """check for commit arguments that aren't committable"""
         if match.isexact() or match.prefix():

mercurial/revlog.py

@@ -302,6 +302,7 b' class revlog:'
         persistentnodemap=False,
         concurrencychecker=None,
         trypending=False,
+        try_split=False,
         canonical_parent_order=True,
     ):
         """
@@ -328,6 +329,7 b' class revlog:'
         self._nodemap_file = None
         self.postfix = postfix
         self._trypending = trypending
+        self._try_split = try_split
         self.opener = opener
         if persistentnodemap:
             self._nodemap_file = nodemaputil.get_nodemap_file(self)
@@ -511,6 +513,8 b' class revlog:'
             entry_point = b'%s.i.%s' % (self.radix, self.postfix)
         elif self._trypending and self.opener.exists(b'%s.i.a' % self.radix):
             entry_point = b'%s.i.a' % self.radix
+        elif self._try_split and self.opener.exists(b'%s.i.s' % self.radix):
+            entry_point = b'%s.i.s' % self.radix
         else:
             entry_point = b'%s.i' % self.radix
 
@@ -2015,7 +2019,7 b' class revlog:'
                 raise error.CensoredNodeError(self.display_id, node, text)
             raise
 
-    def _enforceinlinesize(self, tr):
+    def _enforceinlinesize(self, tr, side_write=True):
         """Check if the revlog is too big for inline and convert if so.
 
         This should be called after revisions are added to the revlog. If the
@@ -2032,7 +2036,8 b' class revlog:'
             raise error.RevlogError(
                 _(b"%s not found in the transaction") % self._indexfile
             )
-        trindex = None
+        if troffset:
+            tr.addbackup(self._indexfile, for_offset=True)
         tr.add(self._datafile, 0)
 
         existing_handles = False
@@ -2048,6 +2053,29 b' class revlog:'
             # No need to deal with sidedata writing handle as it is only
             # relevant with revlog-v2 which is never inline, not reaching
             # this code
+        if side_write:
+            old_index_file_path = self._indexfile
+            new_index_file_path = self._indexfile + b'.s'
+            opener = self.opener
+
+            fncache = getattr(opener, 'fncache', None)
+            if fncache is not None:
+                fncache.addignore(new_index_file_path)
+
+            # the "split" index replace the real index when the transaction is finalized
+            def finalize_callback(tr):
+                opener.rename(
+                    new_index_file_path,
+                    old_index_file_path,
+                    checkambig=True,
+                )
+
+            tr.registertmp(new_index_file_path)
+            if self.target[1] is not None:
+                finalize_id = b'000-revlog-split-%d-%s' % self.target
+            else:
+                finalize_id = b'000-revlog-split-%d' % self.target[0]
+            tr.addfinalize(finalize_id, finalize_callback)
 
         new_dfh = self._datafp(b'w+')
         new_dfh.truncate(0)  # drop any potentially existing data
@@ -2055,17 +2083,10 b' class revlog:'
             with self._indexfp() as read_ifh:
                 for r in self:
                     new_dfh.write(self._getsegmentforrevs(r, r, df=read_ifh)[1])
-                    if (
-                        trindex is None
-                        and troffset
-                        <= self.start(r) + r * self.index.entry_size
-                    ):
-                        trindex = r
                 new_dfh.flush()
 
-            if trindex is None:
-                trindex = 0
-
+            if side_write:
+                self._indexfile = new_index_file_path
             with self.__index_new_fp() as fp:
                 self._format_flags &= ~FLAG_INLINE_DATA
                 self._inline = False
@@ -2079,16 +2100,9 b' class revlog:'
                 if self._docket is not None:
                     self._docket.index_end = fp.tell()
 
-                # There is a small transactional race here. If the rename of
-                # the index fails, we should remove the datafile. It is more
-                # important to ensure that the data file is not truncated
-                # when the index is replaced as otherwise data is lost.
-                tr.replace(self._datafile, self.start(trindex))
-
-                # the temp file replace the real index when we exit the context
-                # manager
-
-            tr.replace(self._indexfile, trindex * self.index.entry_size)
+                # If we don't use side-write, the temp file replace the real
+                # index when we exit the context manager
+
             nodemaputil.setup_persistent_nodemap(tr, self)
             self._segmentfile = randomaccessfile.randomaccessfile(
                 self.opener,

mercurial/store.py

@@ -403,7 +403,7 b" REVLOG_FILES_VOLATILE_EXT = (b'.n', b'.n"
 # some exception to the above matching
 #
 # XXX This is currently not in use because of issue6542
-EXCLUDED = re.compile(b'.*undo\.[^/]+\.(nd?|i)$')
+EXCLUDED = re.compile(br'.*undo\.[^/]+\.(nd?|i)$')
 
 
 def is_revlog(f, kind, st):
@@ -603,6 +603,7 b' class fncache:'
     # hence the encodedir/decodedir dance
     def __init__(self, vfs):
         self.vfs = vfs
+        self._ignores = set()
         self.entries = None
         self._dirty = False
         # set of new additions to fncache
@@ -687,7 +688,12 b' class fncache:'
             self.entries = None
             self.addls = set()
 
+    def addignore(self, fn):
+        self._ignores.add(fn)
+
     def add(self, fn):
+        if fn in self._ignores:
+            return
         if self.entries is None:
             self._load()
         if fn not in self.entries:

mercurial/transaction.py

@@ -105,7 +105,48 b' def _playback('
     unlink=True,
     checkambigfiles=None,
 ):
+    """rollback a transaction :
+    - truncate files that have been appended to
+    - restore file backups
+    - delete temporary files
+    """
+    backupfiles = []
+
+    def restore_one_backup(vfs, f, b, checkambig):
+        filepath = vfs.join(f)
+        backuppath = vfs.join(b)
+        try:
+            util.copyfile(backuppath, filepath, checkambig=checkambig)
+            backupfiles.append((vfs, b))
+        except IOError as exc:
+            e_msg = stringutil.forcebytestr(exc)
+            report(_(b"failed to recover %s (%s)\n") % (f, e_msg))
+            raise
+
+    # gather all backup files that impact the store
+    # (we need this to detect files that are both backed up and truncated)
+    store_backup = {}
+    for entry in backupentries:
+        location, file_path, backup_path, cache = entry
+        vfs = vfsmap[location]
+        is_store = vfs.join(b'') == opener.join(b'')
+        if is_store and file_path and backup_path:
+            store_backup[file_path] = entry
+    copy_done = set()
+
+    # truncate all file `f` to offset `o`
     for f, o in sorted(dict(entries).items()):
+        # if we have a backup for `f`, we should restore it first and truncate
+        # the restored file
+        bck_entry = store_backup.get(f)
+        if bck_entry is not None:
+            location, file_path, backup_path, cache = bck_entry
+            checkambig = False
+            if checkambigfiles:
+                checkambig = (file_path, location) in checkambigfiles
+            restore_one_backup(opener, file_path, backup_path, checkambig)
+            copy_done.add(bck_entry)
+        # truncate the file to its pre-transaction size
         if o or not unlink:
             checkambig = checkambigfiles and (f, b'') in checkambigfiles
             try:
@@ -124,45 +165,52 b' def _playback('
                 report(_(b"failed to truncate %s\n") % f)
                 raise
         else:
+            # delete empty file
             try:
                 opener.unlink(f)
             except FileNotFoundError:
                 pass
-
-    backupfiles = []
-    for l, f, b, c in backupentries:
+    # restore backed up files and clean up temporary files
+    for entry in backupentries:
+        if entry in copy_done:
+            continue
+        l, f, b, c = entry
         if l not in vfsmap and c:
             report(b"couldn't handle %s: unknown cache location %s\n" % (b, l))
         vfs = vfsmap[l]
         try:
+            checkambig = checkambigfiles and (f, l) in checkambigfiles
             if f and b:
-                filepath = vfs.join(f)
-                backuppath = vfs.join(b)
-                checkambig = checkambigfiles and (f, l) in checkambigfiles
-                try:
-                    util.copyfile(backuppath, filepath, checkambig=checkambig)
-                    backupfiles.append(b)
-                except IOError as exc:
-                    e_msg = stringutil.forcebytestr(exc)
-                    report(_(b"failed to recover %s (%s)\n") % (f, e_msg))
+                restore_one_backup(vfs, f, b, checkambig)
             else:
                 target = f or b
                 try:
                     vfs.unlink(target)
                 except FileNotFoundError:
+                    # This is fine because
+                    #
+                    # either we are trying to delete the main file, and it is
+                    # already deleted.
+                    #
+                    # or we are trying to delete a temporary file and it is
+                    # already deleted.
+                    #
+                    # in both case, our target result (delete the file) is
+                    # already achieved.
                     pass
         except (IOError, OSError, error.Abort):
             if not c:
                 raise
 
+    # cleanup transaction state file and the backups file
     backuppath = b"%s.backupfiles" % journal
     if opener.exists(backuppath):
         opener.unlink(backuppath)
     opener.unlink(journal)
     try:
-        for f in backupfiles:
-            if opener.exists(f):
-                opener.unlink(f)
+        for vfs, f in backupfiles:
+            if vfs.exists(f):
+                vfs.unlink(f)
     except (IOError, OSError, error.Abort):
         # only pure backup file remains, it is sage to ignore any error
         pass
@@ -331,7 +379,7 b' class transaction(util.transactional):'
         self._file.flush()
 
     @active
-    def addbackup(self, file, hardlink=True, location=b''):
+    def addbackup(self, file, hardlink=True, location=b'', for_offset=False):
         """Adds a backup of the file to the transaction
 
         Calling addbackup() creates a hardlink backup of the specified file
@@ -340,17 +388,25 b' class transaction(util.transactional):'
 
         * `file`: the file path, relative to .hg/store
         * `hardlink`: use a hardlink to quickly create the backup
+
+        If `for_offset` is set, we expect a offset for this file to have been previously recorded
         """
         if self._queue:
             msg = b'cannot use transaction.addbackup inside "group"'
             raise error.ProgrammingError(msg)
 
-        if (
-            file in self._newfiles
-            or file in self._offsetmap
-            or file in self._backupmap
-        ):
+        if file in self._newfiles or file in self._backupmap:
+            return
+        elif file in self._offsetmap and not for_offset:
             return
+        elif for_offset and file not in self._offsetmap:
+            msg = (
+                'calling `addbackup` with `for_offmap=True`, '
+                'but no offset recorded: [%r] %r'
+            )
+            msg %= (location, file)
+            raise error.ProgrammingError(msg)
+
         vfs = self._vfsmap[location]
         dirname, filename = vfs.split(file)
         backupfilename = b"%s.backup.%s" % (self._journal, filename)

mercurial/url.py

@@ -327,7 +327,9 b' if has_https:'
             self.cert_file = cert_file
 
         def connect(self):
-            self.sock = socket.create_connection((self.host, self.port))
+            self.sock = socket.create_connection(
+                (self.host, self.port), self.timeout
+            )
 
             host = self.host
             realhostport = self.realhostport  # pytype: disable=attribute-error

relnotes/6.4

@@ -1,4 +1,4 b''
-= Mercurial 6.4rc0 =
+= Mercurial 6.4 =
 
 == New Features ==
 
@@ -90,6 +90,31 b' Aside from the following (unordered) com'
  * bundlerepo: apply phase data stored in the bundle instead of assuming `draft`
  * config-item: declare undeclared path suboption
  * narrow: read pending file when applicable
+ * rust: fix building on macOS (issue6801)
+ * run-tests: fix a crash when using the coverage options
+ * undo-files: also remove the undo.backupfiles
+ * undo-files: cleanup backup when cleaning undos
+ * undo-files: clean existing files up before writing new one
+ * undo-files: cleanup legacy files when applicable
+ * dirstate-v2: fix an incorrect handling of readdir errors
+ * rust: update zstd dependency
+ * rust: upgrade `rayon` dependency
+ * dirstate: fix the bug in [status] dealing with committed&ignored directories
+ * dirstate: fix a potential traceback when in `copy` and `rename`
+ * histedit: fix diff colors
+ * cext: fix for PyLong refactoring in CPython 3.12
+ * py3: fix for Python 3.12 emitting SyntaxWarning on invalid escape sequences
+ * statprof: with Python 3.12, lineno is (more) often None
+ * transaction: properly clean up backup file outside of .hg/store/
+ * transaction: raise on backup restoration error
+ * revlog: improve the robustness of the splitting process
+ * debugdeltachain: stop summing the same chain over and over
+ * url: don't ignore timeout for https connections
+ * py3: fix for Python 3.12 emitting SyntaxWarning on invalid escape sequences
+ * tests: accept a test output change in [tests/test-serve.t]
+ * rust: fix thread cap (for real this time)
+ * dirstate: try refreshing the changelog when parent are unknown
+ * hooks: invalidate the repo after the hooks
 
 == Backwards Compatibility Changes ==
  * rust: upgrade supported Rust toolchain version

rust/hg-core/src/dirstate_tree/status.rs

@@ -47,16 +47,10 b" pub fn status<'dirstate>("
     options: StatusOptions,
 ) -> Result<(DirstateStatus<'dirstate>, Vec<PatternFileWarning>), StatusError>
 {
-    // Force the global rayon threadpool to not exceed 16 concurrent threads.
-    // This is a stop-gap measure until we figure out why using more than 16
-    // threads makes `status` slower for each additional thread.
-    // We use `ok()` in case the global threadpool has already been
-    // instantiated in `rhg` or some other caller.
-    // TODO find the underlying cause and fix it, then remove this.
-    rayon::ThreadPoolBuilder::new()
-        .num_threads(16.min(rayon::current_num_threads()))
-        .build_global()
-        .ok();
+    // Also cap for a Python caller of this function, but don't complain if
+    // the global threadpool has already been set since this code path is also
+    // being used by `rhg`, which calls this early.
+    let _ = crate::utils::cap_default_rayon_threads();
 
     let (ignore_fn, warnings, patterns_changed): (IgnoreFnType, _, _) =
         if options.list_ignored || options.list_unknown {

rust/hg-core/src/utils.rs

@@ -498,3 +498,35 b' where'
         Err(e) => Some(Err(e)),
     })
 }
+
+/// Force the global rayon threadpool to not exceed 16 concurrent threads
+/// unless the user has specified a value.
+/// This is a stop-gap measure until we figure out why using more than 16
+/// threads makes `status` slower for each additional thread.
+///
+/// TODO find the underlying cause and fix it, then remove this.
+///
+/// # Errors
+///
+/// Returns an error if the global threadpool has already been initialized if
+/// we try to initialize it.
+pub fn cap_default_rayon_threads() -> Result<(), rayon::ThreadPoolBuildError> {
+    const THREAD_CAP: usize = 16;
+
+    if std::env::var("RAYON_NUM_THREADS").is_err() {
+        let available_parallelism = std::thread::available_parallelism()
+            .map(usize::from)
+            .unwrap_or(1);
+        let new_thread_count = THREAD_CAP.min(available_parallelism);
+        let res = rayon::ThreadPoolBuilder::new()
+            .num_threads(new_thread_count)
+            .build_global();
+        if res.is_ok() {
+            log::trace!(
+                "Capped the rayon threadpool to {new_thread_count} threads",
+            );
+        }
+        return res;
+    }
+    Ok(())
+}

rust/rhg/src/main.rs

@@ -140,6 +140,13 b' fn rhg_main(argv: Vec<OsString>) -> ! {'
 
     env_logger::init();
 
+    // Make sure nothing in a future version of `rhg` sets the global
+    // threadpool before we can cap default threads. (This is also called
+    // in core because Python uses the same code path, we're adding a
+    // redundant check.)
+    hg::utils::cap_default_rayon_threads()
+        .expect("Rayon threadpool already initialized");
+
     let early_args = EarlyArgs::parse(&argv);
 
     let initial_current_dir = early_args.cwd.map(|cwd| {

tests/test-dirstate-read-race.t

@@ -249,7 +249,7 b' commit was created, and status is now cl'
 
 The status process should return a consistent result and not crash.
 
-#if rust no-rhg dirstate-v2-append
+#if no-rhg
   $ cat $TESTTMP/status-race-lock.out
   A dir/o
   R dir/nested/m
@@ -258,7 +258,7 b' The status process should return a consi'
   ? q
   $ cat $TESTTMP/status-race-lock.log
 #else
-#if rhg pre-some-read dirstate-v2-append
+#if pre-some-read dirstate-v2-append
   $ cat $TESTTMP/status-race-lock.out
   A dir/o
   R dir/nested/m
@@ -268,12 +268,10 b' The status process should return a consi'
   $ cat $TESTTMP/status-race-lock.log
 #else
   $ cat $TESTTMP/status-race-lock.out
-  M dir/o (no-rhg known-bad-output !)
   ? dir/n
   ? p
   ? q
   $ cat $TESTTMP/status-race-lock.log
-  warning: ignoring unknown working parent 02a67a77ee9b! (no-rhg !)
 #endif
 #endif
 

tests/test-hook.t

@@ -1423,3 +1423,41 b' HGPLAIN setting in hooks'
   ### no ######## plain: <unset>
   ### auto ###### plain: 1
   Mercurial Distributed SCM (*) (glob)
+
+Test hook that change the underlying repo
+=========================================
+
+blackbox access the dirstate afterward and can see a changelog / dirstate
+desync.
+
+
+  $ cd $TESTTMP
+  $ cat <<EOF >> $HGRCPATH
+  > [extensions]
+  > blackbox=
+  > [hooks]
+  > post-merge = hg commit -m "auto merge"
+  > EOF
+
+  $ hg init t
+  $ cd t
+  $ touch ".hgignore"
+  $ hg commit -Am "initial" -d'0 0'
+  adding .hgignore
+
+  $ echo This is file a1 > a
+  $ hg commit -Am "commit #1" -d'0 0'
+  adding a
+
+  $ hg update 0
+  0 files updated, 0 files merged, 1 files removed, 0 files unresolved
+  $ echo This is file b1 > b
+  $ hg commit -Am "commit #2" -d'0 0'
+  adding b
+  created new head
+
+  $ hg merge 1
+  1 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  (branch merge, don't forget to commit)
+
+  $ cd ..

tests/test-serve.t

@@ -55,7 +55,7 b' With -v and -p daytime'
 #if no-windows
   $ KILLQUIETLY=Y
   $ hgserve -p daytime
-  abort: cannot start server at 'localhost:13': Permission denied (?)
+  abort: cannot start server at 'localhost:13': $EACCES$ (?)
   abort: child process failed to start (?)
   abort: no port number associated with service 'daytime' (?)
   listening at http://localhost/ (bound to $LOCALIP:13) (?)

tests/test-transaction-rollback-on-revlog-split.t

@@ -1,22 +1,103 b''
 Test correctness of revlog inline -> non-inline transition
 ----------------------------------------------------------
 
-Helper extension to intercept renames.
+Helper extension to intercept renames and kill process
 
-  $ cat > $TESTTMP/intercept_rename.py << EOF
+  $ cat > $TESTTMP/intercept_before_rename.py << EOF
   > import os
-  > import sys
+  > import signal
+  > from mercurial import extensions, util
+  > 
+  > def extsetup(ui):
+  >     def rename(orig, src, dest, *args, **kwargs):
+  >         path = util.normpath(dest)
+  >         if path.endswith(b'data/file.i'):
+  >             os.kill(os.getpid(), signal.SIGKILL)
+  >         return orig(src, dest, *args, **kwargs)
+  >     extensions.wrapfunction(util, 'rename', rename)
+  > EOF
+
+  $ cat > $TESTTMP/intercept_after_rename.py << EOF
+  > import os
+  > import signal
   > from mercurial import extensions, util
   > 
   > def extsetup(ui):
   >     def close(orig, *args, **kwargs):
   >         path = util.normpath(args[0]._atomictempfile__name)
+  >         r = orig(*args, **kwargs)
   >         if path.endswith(b'/.hg/store/data/file.i'):
-  >             os._exit(80)
-  >         return orig(*args, **kwargs)
+  >             os.kill(os.getpid(), signal.SIGKILL)
+  >         return r
   >     extensions.wrapfunction(util.atomictempfile, 'close', close)
+  > def extsetup(ui):
+  >     def rename(orig, src, dest, *args, **kwargs):
+  >         path = util.normpath(dest)
+  >         r = orig(src, dest, *args, **kwargs)
+  >         if path.endswith(b'data/file.i'):
+  >             os.kill(os.getpid(), signal.SIGKILL)
+  >         return r
+  >     extensions.wrapfunction(util, 'rename', rename)
   > EOF
 
+  $ cat > $TESTTMP/killme.py << EOF
+  > import os
+  > import signal
+  > 
+  > def killme(ui, repo, hooktype, **kwargs):
+  >     os.kill(os.getpid(), signal.SIGKILL)
+  > EOF
+
+  $ cat > $TESTTMP/reader_wait_split.py << EOF
+  > import os
+  > import signal
+  > from mercurial import extensions, revlog, testing
+  > def _wait_post_load(orig, self, *args, **kwargs):
+  >     wait = b'data/file' in self.radix
+  >     if wait:
+  >         testing.wait_file(b"$TESTTMP/writer-revlog-split")
+  >     r = orig(self, *args, **kwargs)
+  >     if wait:
+  >         testing.write_file(b"$TESTTMP/reader-index-read")
+  >         testing.wait_file(b"$TESTTMP/writer-revlog-unsplit")
+  >     return r
+  > 
+  > def extsetup(ui):
+  >     extensions.wrapfunction(revlog.revlog, '_loadindex', _wait_post_load)
+  > EOF
+
+setup a repository for tests
+----------------------------
+
+  $ cat >> $HGRCPATH << EOF
+  > [format]
+  > revlog-compression=none
+  > EOF
+
+  $ hg init troffset-computation
+  $ cd troffset-computation
+  $ printf '%20d' '1' > file
+  $ hg commit -Aqma
+  $ printf '%1024d' '1' > file
+  $ hg commit -Aqmb
+  $ printf '%20d' '1' > file
+  $ hg commit -Aqmc
+  $ dd if=/dev/zero of=file bs=1k count=128 > /dev/null 2>&1
+  $ hg commit -AqmD --traceback
+
+Reference size:
+  $ f -s file
+  file: size=131072
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.d: size=132139
+  .hg/store/data/file.i: size=256
+
+  $ cd ..
+
+
+Test a hard crash after the file was split but before the transaction was committed
+===================================================================================
+
 Test offset computation to correctly factor in the index entries themselves.
 Also test that the new data size has the correct size if the transaction is aborted
 after the index has been replaced.
@@ -28,30 +109,19 b' and will transition to non-inline storag'
 If the transaction adding c, D is rolled back, then we don't undo the revlog split,
 but truncate the index and the data to remove both c and D.
 
-  $ hg init troffset-computation --config format.revlog-compression=none
-  $ cd troffset-computation
-  $ printf '%20d' '1' > file
-  $ hg commit -Aqma
-  $ printf '%1024d' '1' > file
-  $ hg commit -Aqmb
-  $ printf '%20d' '1' > file
-  $ hg commit -Aqmc
-  $ dd if=/dev/zero of=file bs=1k count=128 > /dev/null 2>&1
-  $ hg commit -AqmD
 
-  $ cd ..
-
-  $ hg clone -r 1 troffset-computation troffset-computation-copy --config format.revlog-compression=none -q
+  $ hg clone --quiet --rev 1 troffset-computation troffset-computation-copy
   $ cd troffset-computation-copy
 
 Reference size:
-
+  $ f -s file
+  file: size=1024
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
 
   $ cat > .hg/hgrc <<EOF
   > [hooks]
-  > pretxnchangegroup = python:$TESTDIR/helper-killhook.py:killme
+  > pretxnchangegroup = python:$TESTTMP/killme.py:killme
   > EOF
 #if chg
   $ hg pull ../troffset-computation
@@ -60,27 +130,38 b' Reference size:'
 #else
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
-  [80]
+  Killed
+  [137]
 #endif
-  $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file | tail -1
-  data/file.i 128
+
+
+The inline revlog still exist, but a split version exist next to it
+
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.d: size=132139
+  .hg/store/data/file.i: size=132395
+  .hg/store/data/file.i.s: size=256
+
 
 The first file.i entry should match the "Reference size" above.
 The first file.d entry is the temporary record during the split,
-the second entry after the split happened. The sum of the second file.d
-and the second file.i entry should match the first file.i entry.
+
+A "temporary file" entry exist for the split index.
 
   $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file
   data/file.i 1174
   data/file.d 0
-  data/file.d 1046
-  data/file.i 128
+  $ cat .hg/store/journal.backupfiles | tr -s '\000' ' ' | tr -s '\00' ' '| grep data/file
+   data/file.i data/journal.backup.file.i 0
+   data/file.i.s 0
+
+recover is rolling the split back, the fncache is still valid
+
   $ hg recover
   rolling back interrupted transaction
   (verify step skipped, run `hg verify` to check your repository content)
   $ f -s .hg/store/data/file*
-  .hg/store/data/file.d: size=1046
-  .hg/store/data/file.i: size=128
+  .hg/store/data/file.i: size=1174
   $ hg tip
   changeset:   1:cfa8d6e60429
   tag:         tip
@@ -89,47 +170,67 b' and the second file.i entry should match'
   summary:     b
   
   $ hg verify -q
-   warning: revlog 'data/file.d' not in fncache!
-  1 warnings encountered!
-  hint: run "hg debugrebuildfncache" to recover from corrupt fncache
   $ hg debugrebuildfncache --only-data
-  adding data/file.d
-  1 items added, 0 removed from fncache
+  fncache already up to date
   $ hg verify -q
   $ cd ..
 
+Test a hard crash right before the index is move into place
+===========================================================
 
 Now retry the procedure but intercept the rename of the index and check that
 the journal does not contain the new index size. This demonstrates the edge case
 where the data file is left as garbage.
 
-  $ hg clone -r 1 troffset-computation troffset-computation-copy2 --config format.revlog-compression=none -q
+  $ hg clone --quiet --rev 1 troffset-computation troffset-computation-copy2
   $ cd troffset-computation-copy2
+
+Reference size:
+  $ f -s file
+  file: size=1024
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.i: size=1174
+
   $ cat > .hg/hgrc <<EOF
   > [extensions]
-  > intercept_rename = $TESTTMP/intercept_rename.py
-  > [hooks]
-  > pretxnchangegroup = python:$TESTDIR/helper-killhook.py:killme
+  > intercept_rename = $TESTTMP/intercept_before_rename.py
   > EOF
 #if chg
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
   [255]
 #else
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
-  [80]
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  Killed
+  [137]
 #endif
+
+The inline revlog still exist, but a split version exist next to it
+
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.d: size=132139
+  .hg/store/data/file.i: size=132395
+  .hg/store/data/file.i.s: size=256
+
   $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file
   data/file.i 1174
   data/file.d 0
-  data/file.d 1046
+
+recover is rolling the split back, the fncache is still valid
 
   $ hg recover
   rolling back interrupted transaction
   (verify step skipped, run `hg verify` to check your repository content)
   $ f -s .hg/store/data/file*
-  .hg/store/data/file.d: size=1046
   .hg/store/data/file.i: size=1174
   $ hg tip
   changeset:   1:cfa8d6e60429
@@ -141,10 +242,77 b' where the data file is left as garbage.'
   $ hg verify -q
   $ cd ..
 
+Test a hard crash right after the index is move into place
+===========================================================
+
+Now retry the procedure but intercept the rename of the index.
+
+  $ hg clone --quiet --rev 1 troffset-computation troffset-computation-crash-after-rename
+  $ cd troffset-computation-crash-after-rename
+
+Reference size:
+  $ f -s file
+  file: size=1024
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.i: size=1174
+
+  $ cat > .hg/hgrc <<EOF
+  > [extensions]
+  > intercept_rename = $TESTTMP/intercept_after_rename.py
+  > EOF
+#if chg
+  $ hg pull ../troffset-computation
+  pulling from ../troffset-computation
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  [255]
+#else
+  $ hg pull ../troffset-computation
+  pulling from ../troffset-computation
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  Killed
+  [137]
+#endif
+
+The inline revlog was over written on disk
+
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.d: size=132139
+  .hg/store/data/file.i: size=256
+
+  $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file
+  data/file.i 1174
+  data/file.d 0
+
+recover is rolling the split back, the fncache is still valid
+
+  $ hg recover
+  rolling back interrupted transaction
+  (verify step skipped, run `hg verify` to check your repository content)
+  $ f -s .hg/store/data/file*
+  .hg/store/data/file.i: size=1174
+  $ hg tip
+  changeset:   1:cfa8d6e60429
+  tag:         tip
+  user:        test
+  date:        Thu Jan 01 00:00:00 1970 +0000
+  summary:     b
+  
+  $ hg verify -q
+  $ cd ..
+
+Have the transaction rollback itself without any hard crash
+===========================================================
+
 
 Repeat the original test but let hg rollback the transaction.
 
-  $ hg clone -r 1 troffset-computation troffset-computation-copy-rb --config format.revlog-compression=none -q
+  $ hg clone --quiet --rev 1 troffset-computation troffset-computation-copy-rb
   $ cd troffset-computation-copy-rb
   $ cat > .hg/hgrc <<EOF
   > [hooks]
@@ -160,9 +328,14 b' Repeat the original test but let hg roll'
   rollback completed
   abort: pretxnchangegroup hook exited with status 1
   [40]
+
+The split was rollback
+
   $ f -s .hg/store/data/file*
-  .hg/store/data/file.d: size=1046
-  .hg/store/data/file.i: size=128
+  .hg/store/data/file.d: size=0
+  .hg/store/data/file.i: size=1174
+
+
   $ hg tip
   changeset:   1:cfa8d6e60429
   tag:         tip
@@ -171,8 +344,85 b' Repeat the original test but let hg roll'
   summary:     b
   
   $ hg verify -q
-   warning: revlog 'data/file.d' not in fncache!
-  1 warnings encountered!
-  hint: run "hg debugrebuildfncache" to recover from corrupt fncache
   $ cd ..
 
+Read race
+=========
+
+We check that a client that started reading a revlog (its index) after the
+split and end reading (the data) after the rollback should be fine
+
+  $ hg clone --quiet --rev 1 troffset-computation troffset-computation-race
+  $ cd troffset-computation-race
+  $ cat > .hg/hgrc <<EOF
+  > [hooks]
+  > pretxnchangegroup=$RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/reader-index-read $TESTTMP/writer-revlog-split
+  > pretxnclose = false
+  > EOF
+
+start a reader
+
+  $ hg cat --rev 0 file \
+  > --config "extensions.wait_read=$TESTTMP/reader_wait_split.py" \
+  > 2> $TESTTMP/reader.stderr \
+  > > $TESTTMP/reader.stdout &
+
+Do a failed pull in //
+
+  $ hg pull ../troffset-computation
+  pulling from ../troffset-computation
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  transaction abort!
+  rollback completed
+  abort: pretxnclose hook exited with status 1
+  [40]
+  $ touch $TESTTMP/writer-revlog-unsplit
+  $ wait
+
+The reader should be fine
+  $ cat $TESTTMP/reader.stderr
+  $ cat $TESTTMP/reader.stdout
+                     1 (no-eol)
+  $ cd ..
+
+pending hooks
+=============
+
+We checks that hooks properly see the inside of the transaction, while other process don't.
+
+  $ hg clone --quiet --rev 1 troffset-computation troffset-computation-hooks
+  $ cd troffset-computation-hooks
+  $ cat > .hg/hgrc <<EOF
+  > [hooks]
+  > pretxnclose.01-echo = hg cat -r 'max(all())' file | f --size
+  > pretxnclose.02-echo = $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/hook-done $TESTTMP/hook-tr-ready
+  > pretxnclose.03-abort = false
+  > EOF
+
+  $ (
+  >   $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/hook-tr-ready;\
+  >   hg cat -r 'max(all())' file | f --size;\
+  >   touch $TESTTMP/hook-done
+  > ) >stdout 2>stderr &
+
+  $ hg pull ../troffset-computation
+  pulling from ../troffset-computation
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  size=131072
+  transaction abort!
+  rollback completed
+  abort: pretxnclose.03-abort hook exited with status 1
+  [40]
+
+  $ cat stdout
+  size=1024
+  $ cat stderr
+
+
+  $ cd ..