backup: fix issue when the backup end up in a different directory

Because of store encoding, we might end up with the backup in a different
directory than the initial copy (for example if the backup path make it cross
the 120 char limit).

This can create crash, especially since 6.4 where backup are used during revlog
split.

Making sure the directory exists fixes these crash We added a test covering this
case.

Strictly speaking, this has always been broken, however the new code in 6.4
triggers it more easily.

mercurial/transaction.py

@@ -1,955 +1,960 b''
 # transaction.py - simple journaling scheme for mercurial
 #
 # This transaction scheme is intended to gracefully handle program
 # errors and interruptions. More serious failures like system crashes
 # can be recovered with an fsck-like tool. As the whole repository is
 # effectively log-structured, this should amount to simply truncating
 # anything that isn't referenced in the changelog.
 #
 # Copyright 2005, 2006 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import errno
 import os
 
 from .i18n import _
 from . import (
     error,
     pycompat,
     util,
 )
 from .utils import stringutil
 
 version = 2
 
 GEN_GROUP_ALL = b'all'
 GEN_GROUP_PRE_FINALIZE = b'prefinalize'
 GEN_GROUP_POST_FINALIZE = b'postfinalize'
 
 
 def active(func):
     def _active(self, *args, **kwds):
         if self._count == 0:
             raise error.ProgrammingError(
                 b'cannot use transaction when it is already committed/aborted'
             )
         return func(self, *args, **kwds)
 
     return _active
 
 
 UNDO_BACKUP = b'%s.backupfiles'
 
 UNDO_FILES_MAY_NEED_CLEANUP = [
     # legacy entries that might exists on disk from previous version:
     (b'store', b'%s.narrowspec'),
     (b'plain', b'%s.narrowspec.dirstate'),
     (b'plain', b'%s.branch'),
     (b'plain', b'%s.bookmarks'),
     (b'store', b'%s.phaseroots'),
     (b'plain', b'%s.dirstate'),
     # files actually in uses today:
     (b'plain', b'%s.desc'),
     # Always delete undo last to make sure we detect that a clean up is needed if
     # the process is interrupted.
     (b'store', b'%s'),
 ]
 
 
 def cleanup_undo_files(report, vfsmap, undo_prefix=b'undo'):
     """remove "undo" files used by the rollback logic
 
     This is useful to prevent rollback running in situation were it does not
     make sense. For example after a strip.
     """
     backup_listing = UNDO_BACKUP % undo_prefix
 
     backup_entries = []
     undo_files = []
     svfs = vfsmap[b'store']
     try:
         with svfs(backup_listing) as f:
             backup_entries = read_backup_files(report, f)
     except OSError as e:
         if e.errno != errno.ENOENT:
             msg = _(b'could not read %s: %s\n')
             msg %= (svfs.join(backup_listing), stringutil.forcebytestr(e))
             report(msg)
 
     for location, f, backup_path, c in backup_entries:
         if location in vfsmap and backup_path:
             undo_files.append((vfsmap[location], backup_path))
 
     undo_files.append((svfs, backup_listing))
     for location, undo_path in UNDO_FILES_MAY_NEED_CLEANUP:
         undo_files.append((vfsmap[location], undo_path % undo_prefix))
     for undovfs, undofile in undo_files:
         try:
             undovfs.unlink(undofile)
         except OSError as e:
             if e.errno != errno.ENOENT:
                 msg = _(b'error removing %s: %s\n')
                 msg %= (undovfs.join(undofile), stringutil.forcebytestr(e))
                 report(msg)
 
 
 def _playback(
     journal,
     report,
     opener,
     vfsmap,
     entries,
     backupentries,
     unlink=True,
     checkambigfiles=None,
 ):
     """rollback a transaction :
     - truncate files that have been appended to
     - restore file backups
     - delete temporary files
     """
     backupfiles = []
 
     def restore_one_backup(vfs, f, b, checkambig):
         filepath = vfs.join(f)
         backuppath = vfs.join(b)
         try:
             util.copyfile(backuppath, filepath, checkambig=checkambig)
             backupfiles.append((vfs, b))
         except IOError as exc:
             e_msg = stringutil.forcebytestr(exc)
             report(_(b"failed to recover %s (%s)\n") % (f, e_msg))
             raise
 
     # gather all backup files that impact the store
     # (we need this to detect files that are both backed up and truncated)
     store_backup = {}
     for entry in backupentries:
         location, file_path, backup_path, cache = entry
         vfs = vfsmap[location]
         is_store = vfs.join(b'') == opener.join(b'')
         if is_store and file_path and backup_path:
             store_backup[file_path] = entry
     copy_done = set()
 
     # truncate all file `f` to offset `o`
     for f, o in sorted(dict(entries).items()):
         # if we have a backup for `f`, we should restore it first and truncate
         # the restored file
         bck_entry = store_backup.get(f)
         if bck_entry is not None:
             location, file_path, backup_path, cache = bck_entry
             checkambig = False
             if checkambigfiles:
                 checkambig = (file_path, location) in checkambigfiles
             restore_one_backup(opener, file_path, backup_path, checkambig)
             copy_done.add(bck_entry)
         # truncate the file to its pre-transaction size
         if o or not unlink:
             checkambig = checkambigfiles and (f, b'') in checkambigfiles
             try:
                 fp = opener(f, b'a', checkambig=checkambig)
                 if fp.tell() < o:
                     raise error.Abort(
                         _(
                             b"attempted to truncate %s to %d bytes, but it was "
                             b"already %d bytes\n"
                         )
                         % (f, o, fp.tell())
                     )
                 fp.truncate(o)
                 fp.close()
             except IOError:
                 report(_(b"failed to truncate %s\n") % f)
                 raise
         else:
             # delete empty file
             try:
                 opener.unlink(f)
             except FileNotFoundError:
                 pass
     # restore backed up files and clean up temporary files
     for entry in backupentries:
         if entry in copy_done:
             continue
         l, f, b, c = entry
         if l not in vfsmap and c:
             report(b"couldn't handle %s: unknown cache location %s\n" % (b, l))
         vfs = vfsmap[l]
         try:
             checkambig = checkambigfiles and (f, l) in checkambigfiles
             if f and b:
                 restore_one_backup(vfs, f, b, checkambig)
             else:
                 target = f or b
                 try:
                     vfs.unlink(target)
                 except FileNotFoundError:
                     # This is fine because
                     #
                     # either we are trying to delete the main file, and it is
                     # already deleted.
                     #
                     # or we are trying to delete a temporary file and it is
                     # already deleted.
                     #
                     # in both case, our target result (delete the file) is
                     # already achieved.
                     pass
         except (IOError, OSError, error.Abort):
             if not c:
                 raise
 
     # cleanup transaction state file and the backups file
     backuppath = b"%s.backupfiles" % journal
     if opener.exists(backuppath):
         opener.unlink(backuppath)
     opener.unlink(journal)
     try:
         for vfs, f in backupfiles:
             if vfs.exists(f):
                 vfs.unlink(f)
     except (IOError, OSError, error.Abort):
         # only pure backup file remains, it is sage to ignore any error
         pass
 
 
 class transaction(util.transactional):
     def __init__(
         self,
         report,
         opener,
         vfsmap,
         journalname,
         undoname=None,
         after=None,
         createmode=None,
         validator=None,
         releasefn=None,
         checkambigfiles=None,
         name='<unnamed>',
     ):
         """Begin a new transaction
 
         Begins a new transaction that allows rolling back writes in the event of
         an exception.
 
         * `after`: called after the transaction has been committed
         * `createmode`: the mode of the journal file that will be created
         * `releasefn`: called after releasing (with transaction and result)
 
         `checkambigfiles` is a set of (path, vfs-location) tuples,
         which determine whether file stat ambiguity should be avoided
         for corresponded files.
         """
         self._count = 1
         self._usages = 1
         self._report = report
         # a vfs to the store content
         self._opener = opener
         # a map to access file in various {location -> vfs}
         vfsmap = vfsmap.copy()
         vfsmap[b''] = opener  # set default value
         self._vfsmap = vfsmap
         self._after = after
         self._offsetmap = {}
         self._newfiles = set()
         self._journal = journalname
         self._journal_files = []
         self._undoname = undoname
         self._queue = []
         # A callback to do something just after releasing transaction.
         if releasefn is None:
             releasefn = lambda tr, success: None
         self._releasefn = releasefn
 
         self._checkambigfiles = set()
         if checkambigfiles:
             self._checkambigfiles.update(checkambigfiles)
 
         self._names = [name]
 
         # A dict dedicated to precisely tracking the changes introduced in the
         # transaction.
         self.changes = {}
 
         # a dict of arguments to be passed to hooks
         self.hookargs = {}
         self._file = opener.open(self._journal, b"w+")
 
         # a list of ('location', 'path', 'backuppath', cache) entries.
         # - if 'backuppath' is empty, no file existed at backup time
         # - if 'path' is empty, this is a temporary transaction file
         # - if 'location' is not empty, the path is outside main opener reach.
         #   use 'location' value as a key in a vfsmap to find the right 'vfs'
         # (cache is currently unused)
         self._backupentries = []
         self._backupmap = {}
         self._backupjournal = b"%s.backupfiles" % self._journal
         self._backupsfile = opener.open(self._backupjournal, b'w')
         self._backupsfile.write(b'%d\n' % version)
 
         if createmode is not None:
             opener.chmod(self._journal, createmode & 0o666)
             opener.chmod(self._backupjournal, createmode & 0o666)
 
         # hold file generations to be performed on commit
         self._filegenerators = {}
         # hold callback to write pending data for hooks
         self._pendingcallback = {}
         # True is any pending data have been written ever
         self._anypending = False
         # holds callback to call when writing the transaction
         self._finalizecallback = {}
         # holds callback to call when validating the transaction
         # should raise exception if anything is wrong
         self._validatecallback = {}
         if validator is not None:
             self._validatecallback[b'001-userhooks'] = validator
         # hold callback for post transaction close
         self._postclosecallback = {}
         # holds callbacks to call during abort
         self._abortcallback = {}
 
     def __repr__(self):
         name = '/'.join(self._names)
         return '<transaction name=%s, count=%d, usages=%d>' % (
             name,
             self._count,
             self._usages,
         )
 
     def __del__(self):
         if self._journal:
             self._abort()
 
     @property
     def finalized(self):
         return self._finalizecallback is None
 
     @active
     def startgroup(self):
         """delay registration of file entry
 
         This is used by strip to delay vision of strip offset. The transaction
         sees either none or all of the strip actions to be done."""
         self._queue.append([])
 
     @active
     def endgroup(self):
         """apply delayed registration of file entry.
 
         This is used by strip to delay vision of strip offset. The transaction
         sees either none or all of the strip actions to be done."""
         q = self._queue.pop()
         for f, o in q:
             self._addentry(f, o)
 
     @active
     def add(self, file, offset):
         """record the state of an append-only file before update"""
         if (
             file in self._newfiles
             or file in self._offsetmap
             or file in self._backupmap
         ):
             return
         if self._queue:
             self._queue[-1].append((file, offset))
             return
 
         self._addentry(file, offset)
 
     def _addentry(self, file, offset):
         """add a append-only entry to memory and on-disk state"""
         if (
             file in self._newfiles
             or file in self._offsetmap
             or file in self._backupmap
         ):
             return
         if offset:
             self._offsetmap[file] = offset
         else:
             self._newfiles.add(file)
         # add enough data to the journal to do the truncate
         self._file.write(b"%s\0%d\n" % (file, offset))
         self._file.flush()
 
     @active
     def addbackup(self, file, hardlink=True, location=b'', for_offset=False):
         """Adds a backup of the file to the transaction
 
         Calling addbackup() creates a hardlink backup of the specified file
         that is used to recover the file in the event of the transaction
         aborting.
 
         * `file`: the file path, relative to .hg/store
         * `hardlink`: use a hardlink to quickly create the backup
 
         If `for_offset` is set, we expect a offset for this file to have been previously recorded
         """
         if self._queue:
             msg = b'cannot use transaction.addbackup inside "group"'
             raise error.ProgrammingError(msg)
 
         if file in self._newfiles or file in self._backupmap:
             return
         elif file in self._offsetmap and not for_offset:
             return
         elif for_offset and file not in self._offsetmap:
             msg = (
                 'calling `addbackup` with `for_offmap=True`, '
                 'but no offset recorded: [%r] %r'
             )
             msg %= (location, file)
             raise error.ProgrammingError(msg)
 
         vfs = self._vfsmap[location]
         dirname, filename = vfs.split(file)
         backupfilename = b"%s.backup.%s" % (self._journal, filename)
         backupfile = vfs.reljoin(dirname, backupfilename)
         if vfs.exists(file):
             filepath = vfs.join(file)
             backuppath = vfs.join(backupfile)
+            # store encoding may result in different directory here.
+            # so we have to ensure the destination directory exist
+            final_dir_name = os.path.dirname(backuppath)
+            util.makedirs(final_dir_name, mode=vfs.createmode, notindexed=True)
+            # then we can copy the backup
             util.copyfile(filepath, backuppath, hardlink=hardlink)
         else:
             backupfile = b''
 
         self._addbackupentry((location, file, backupfile, False))
 
     def _addbackupentry(self, entry):
         """register a new backup entry and write it to disk"""
         self._backupentries.append(entry)
         self._backupmap[entry[1]] = len(self._backupentries) - 1
         self._backupsfile.write(b"%s\0%s\0%s\0%d\n" % entry)
         self._backupsfile.flush()
 
     @active
     def registertmp(self, tmpfile, location=b''):
         """register a temporary transaction file
 
         Such files will be deleted when the transaction exits (on both
         failure and success).
         """
         self._addbackupentry((location, b'', tmpfile, False))
 
     @active
     def addfilegenerator(
         self,
         genid,
         filenames,
         genfunc,
         order=0,
         location=b'',
         post_finalize=False,
     ):
         """add a function to generates some files at transaction commit
 
         The `genfunc` argument is a function capable of generating proper
         content of each entry in the `filename` tuple.
 
         At transaction close time, `genfunc` will be called with one file
         object argument per entries in `filenames`.
 
         The transaction itself is responsible for the backup, creation and
         final write of such file.
 
         The `genid` argument is used to ensure the same set of file is only
         generated once. Call to `addfilegenerator` for a `genid` already
         present will overwrite the old entry.
 
         The `order` argument may be used to control the order in which multiple
         generator will be executed.
 
         The `location` arguments may be used to indicate the files are located
         outside of the the standard directory for transaction. It should match
         one of the key of the `transaction.vfsmap` dictionary.
 
         The `post_finalize` argument can be set to `True` for file generation
         that must be run after the transaction has been finalized.
         """
         # For now, we are unable to do proper backup and restore of custom vfs
         # but for bookmarks that are handled outside this mechanism.
         entry = (order, filenames, genfunc, location, post_finalize)
         self._filegenerators[genid] = entry
 
     @active
     def removefilegenerator(self, genid):
         """reverse of addfilegenerator, remove a file generator function"""
         if genid in self._filegenerators:
             del self._filegenerators[genid]
 
     def _generatefiles(self, suffix=b'', group=GEN_GROUP_ALL):
         # write files registered for generation
         any = False
 
         if group == GEN_GROUP_ALL:
             skip_post = skip_pre = False
         else:
             skip_pre = group == GEN_GROUP_POST_FINALIZE
             skip_post = group == GEN_GROUP_PRE_FINALIZE
 
         for id, entry in sorted(self._filegenerators.items()):
             any = True
             order, filenames, genfunc, location, post_finalize = entry
 
             # for generation at closing, check if it's before or after finalize
             if skip_post and post_finalize:
                 continue
             elif skip_pre and not post_finalize:
                 continue
 
             vfs = self._vfsmap[location]
             files = []
             try:
                 for name in filenames:
                     name += suffix
                     if suffix:
                         self.registertmp(name, location=location)
                         checkambig = False
                     else:
                         self.addbackup(name, location=location)
                         checkambig = (name, location) in self._checkambigfiles
                     files.append(
                         vfs(name, b'w', atomictemp=True, checkambig=checkambig)
                     )
                 genfunc(*files)
                 for f in files:
                     f.close()
                 # skip discard() loop since we're sure no open file remains
                 del files[:]
             finally:
                 for f in files:
                     f.discard()
         return any
 
     @active
     def findoffset(self, file):
         if file in self._newfiles:
             return 0
         return self._offsetmap.get(file)
 
     @active
     def readjournal(self):
         self._file.seek(0)
         entries = []
         for l in self._file.readlines():
             file, troffset = l.split(b'\0')
             entries.append((file, int(troffset)))
         return entries
 
     @active
     def replace(self, file, offset):
         """
         replace can only replace already committed entries
         that are not pending in the queue
         """
         if file in self._newfiles:
             if not offset:
                 return
             self._newfiles.remove(file)
             self._offsetmap[file] = offset
         elif file in self._offsetmap:
             if not offset:
                 del self._offsetmap[file]
                 self._newfiles.add(file)
             else:
                 self._offsetmap[file] = offset
         else:
             raise KeyError(file)
         self._file.write(b"%s\0%d\n" % (file, offset))
         self._file.flush()
 
     @active
     def nest(self, name='<unnamed>'):
         self._count += 1
         self._usages += 1
         self._names.append(name)
         return self
 
     def release(self):
         if self._count > 0:
             self._usages -= 1
         if self._names:
             self._names.pop()
         # if the transaction scopes are left without being closed, fail
         if self._count > 0 and self._usages == 0:
             self._abort()
 
     def running(self):
         return self._count > 0
 
     def addpending(self, category, callback):
         """add a callback to be called when the transaction is pending
 
         The transaction will be given as callback's first argument.
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._pendingcallback[category] = callback
 
     @active
     def writepending(self):
         """write pending file to temporary version
 
         This is used to allow hooks to view a transaction before commit"""
         categories = sorted(self._pendingcallback)
         for cat in categories:
             # remove callback since the data will have been flushed
             any = self._pendingcallback.pop(cat)(self)
             self._anypending = self._anypending or any
         self._anypending |= self._generatefiles(suffix=b'.pending')
         return self._anypending
 
     @active
     def hasfinalize(self, category):
         """check is a callback already exist for a category"""
         return category in self._finalizecallback
 
     @active
     def addfinalize(self, category, callback):
         """add a callback to be called when the transaction is closed
 
         The transaction will be given as callback's first argument.
 
         Category is a unique identifier to allow overwriting old callbacks with
         newer callbacks.
         """
         self._finalizecallback[category] = callback
 
     @active
     def addpostclose(self, category, callback):
         """add or replace a callback to be called after the transaction closed
 
         The transaction will be given as callback's first argument.
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._postclosecallback[category] = callback
 
     @active
     def getpostclose(self, category):
         """return a postclose callback added before, or None"""
         return self._postclosecallback.get(category, None)
 
     @active
     def addabort(self, category, callback):
         """add a callback to be called when the transaction is aborted.
 
         The transaction will be given as the first argument to the callback.
 
         Category is a unique identifier to allow overwriting an old callback
         with a newer callback.
         """
         self._abortcallback[category] = callback
 
     @active
     def addvalidator(self, category, callback):
         """adds a callback to be called when validating the transaction.
 
         The transaction will be given as the first argument to the callback.
 
         callback should raise exception if to abort transaction"""
         self._validatecallback[category] = callback
 
     @active
     def close(self):
         '''commit the transaction'''
         if self._count == 1:
             for category in sorted(self._validatecallback):
                 self._validatecallback[category](self)
             self._validatecallback = None  # Help prevent cycles.
             self._generatefiles(group=GEN_GROUP_PRE_FINALIZE)
             while self._finalizecallback:
                 callbacks = self._finalizecallback
                 self._finalizecallback = {}
                 categories = sorted(callbacks)
                 for cat in categories:
                     callbacks[cat](self)
             # Prevent double usage and help clear cycles.
             self._finalizecallback = None
             self._generatefiles(group=GEN_GROUP_POST_FINALIZE)
 
         self._count -= 1
         if self._count != 0:
             return
         self._file.close()
         self._backupsfile.close()
         # cleanup temporary files
         for l, f, b, c in self._backupentries:
             if l not in self._vfsmap and c:
                 self._report(
                     b"couldn't remove %s: unknown cache location %s\n" % (b, l)
                 )
                 continue
             vfs = self._vfsmap[l]
             if not f and b and vfs.exists(b):
                 try:
                     vfs.unlink(b)
                 except (IOError, OSError, error.Abort) as inst:
                     if not c:
                         raise
                     # Abort may be raise by read only opener
                     self._report(
                         b"couldn't remove %s: %s\n" % (vfs.join(b), inst)
                     )
         self._offsetmap = {}
         self._newfiles = set()
         self._writeundo()
         if self._after:
             self._after()
             self._after = None  # Help prevent cycles.
         if self._opener.isfile(self._backupjournal):
             self._opener.unlink(self._backupjournal)
         if self._opener.isfile(self._journal):
             self._opener.unlink(self._journal)
         for l, _f, b, c in self._backupentries:
             if l not in self._vfsmap and c:
                 self._report(
                     b"couldn't remove %s: unknown cache location"
                     b"%s\n" % (b, l)
                 )
                 continue
             vfs = self._vfsmap[l]
             if b and vfs.exists(b):
                 try:
                     vfs.unlink(b)
                 except (IOError, OSError, error.Abort) as inst:
                     if not c:
                         raise
                     # Abort may be raise by read only opener
                     self._report(
                         b"couldn't remove %s: %s\n" % (vfs.join(b), inst)
                     )
         self._backupentries = []
         self._journal = None
 
         self._releasefn(self, True)  # notify success of closing transaction
         self._releasefn = None  # Help prevent cycles.
 
         # run post close action
         categories = sorted(self._postclosecallback)
         for cat in categories:
             self._postclosecallback[cat](self)
         # Prevent double usage and help clear cycles.
         self._postclosecallback = None
 
     @active
     def abort(self):
         """abort the transaction (generally called on error, or when the
         transaction is not explicitly committed before going out of
         scope)"""
         self._abort()
 
     @active
     def add_journal(self, vfs_id, path):
         self._journal_files.append((vfs_id, path))
 
     def _writeundo(self):
         """write transaction data for possible future undo call"""
         if self._undoname is None:
             return
         cleanup_undo_files(
             self._report,
             self._vfsmap,
             undo_prefix=self._undoname,
         )
 
         def undoname(fn: bytes) -> bytes:
             base, name = os.path.split(fn)
             assert name.startswith(self._journal)
             new_name = name.replace(self._journal, self._undoname, 1)
             return os.path.join(base, new_name)
 
         undo_backup_path = b"%s.backupfiles" % self._undoname
         undobackupfile = self._opener.open(undo_backup_path, b'w')
         undobackupfile.write(b'%d\n' % version)
         for l, f, b, c in self._backupentries:
             if not f:  # temporary file
                 continue
             if not b:
                 u = b''
             else:
                 if l not in self._vfsmap and c:
                     self._report(
                         b"couldn't remove %s: unknown cache location"
                         b"%s\n" % (b, l)
                     )
                     continue
                 vfs = self._vfsmap[l]
                 u = undoname(b)
                 util.copyfile(vfs.join(b), vfs.join(u), hardlink=True)
             undobackupfile.write(b"%s\0%s\0%s\0%d\n" % (l, f, u, c))
         undobackupfile.close()
         for vfs, src in self._journal_files:
             dest = undoname(src)
             # if src and dest refer to a same file, vfs.rename is a no-op,
             # leaving both src and dest on disk. delete dest to make sure
             # the rename couldn't be such a no-op.
             vfs.tryunlink(dest)
             try:
                 vfs.rename(src, dest)
             except FileNotFoundError:  # journal file does not yet exist
                 pass
 
     def _abort(self):
         entries = self.readjournal()
         self._count = 0
         self._usages = 0
         self._file.close()
         self._backupsfile.close()
 
         quick = self._can_quick_abort(entries)
         try:
             if not quick:
                 self._report(_(b"transaction abort!\n"))
             for cat in sorted(self._abortcallback):
                 self._abortcallback[cat](self)
             # Prevent double usage and help clear cycles.
             self._abortcallback = None
             if quick:
                 self._do_quick_abort(entries)
             else:
                 self._do_full_abort(entries)
         finally:
             self._journal = None
             self._releasefn(self, False)  # notify failure of transaction
             self._releasefn = None  # Help prevent cycles.
 
     def _can_quick_abort(self, entries):
         """False if any semantic content have been written on disk
 
         True if nothing, except temporary files has been writen on disk."""
         if entries:
             return False
         for e in self._backupentries:
             if e[1]:
                 return False
         return True
 
     def _do_quick_abort(self, entries):
         """(Silently) do a quick cleanup (see _can_quick_abort)"""
         assert self._can_quick_abort(entries)
         tmp_files = [e for e in self._backupentries if not e[1]]
         for vfs_id, old_path, tmp_path, xxx in tmp_files:
             vfs = self._vfsmap[vfs_id]
             try:
                 vfs.unlink(tmp_path)
             except FileNotFoundError:
                 pass
         if self._backupjournal:
             self._opener.unlink(self._backupjournal)
         if self._journal:
             self._opener.unlink(self._journal)
 
     def _do_full_abort(self, entries):
         """(Noisily) rollback all the change introduced by the transaction"""
         try:
             _playback(
                 self._journal,
                 self._report,
                 self._opener,
                 self._vfsmap,
                 entries,
                 self._backupentries,
                 False,
                 checkambigfiles=self._checkambigfiles,
             )
             self._report(_(b"rollback completed\n"))
         except BaseException as exc:
             self._report(_(b"rollback failed - please run hg recover\n"))
             self._report(
                 _(b"(failure reason: %s)\n") % stringutil.forcebytestr(exc)
             )
 
 
 BAD_VERSION_MSG = _(
     b"journal was created by a different version of Mercurial\n"
 )
 
 
 def read_backup_files(report, fp):
     """parse an (already open) backup file an return contained backup entries
 
     entries are in the form: (location, file, backupfile, xxx)
 
     :location:   the vfs identifier (vfsmap's key)
     :file:       original file path (in the vfs)
     :backupfile: path of the backup (in the vfs)
     :cache:      a boolean currently always set to False
     """
     lines = fp.readlines()
     backupentries = []
     if lines:
         ver = lines[0][:-1]
         if ver != (b'%d' % version):
             report(BAD_VERSION_MSG)
         else:
             for line in lines[1:]:
                 if line:
                     # Shave off the trailing newline
                     line = line[:-1]
                     l, f, b, c = line.split(b'\0')
                     backupentries.append((l, f, b, bool(c)))
     return backupentries
 
 
 def rollback(
     opener,
     vfsmap,
     file,
     report,
     checkambigfiles=None,
     skip_journal_pattern=None,
 ):
     """Rolls back the transaction contained in the given file
 
     Reads the entries in the specified file, and the corresponding
     '*.backupfiles' file, to recover from an incomplete transaction.
 
     * `file`: a file containing a list of entries, specifying where
     to truncate each file.  The file should contain a list of
     file\0offset pairs, delimited by newlines. The corresponding
     '*.backupfiles' file should contain a list of file\0backupfile
     pairs, delimited by \0.
 
     `checkambigfiles` is a set of (path, vfs-location) tuples,
     which determine whether file stat ambiguity should be avoided at
     restoring corresponded files.
     """
     entries = []
     backupentries = []
 
     with opener.open(file) as fp:
         lines = fp.readlines()
     for l in lines:
         try:
             f, o = l.split(b'\0')
             entries.append((f, int(o)))
         except ValueError:
             report(
                 _(b"couldn't read journal entry %r!\n") % pycompat.bytestr(l)
             )
 
     backupjournal = b"%s.backupfiles" % file
     if opener.exists(backupjournal):
         with opener.open(backupjournal) as fp:
             backupentries = read_backup_files(report, fp)
     if skip_journal_pattern is not None:
         keep = lambda x: not skip_journal_pattern.match(x[1])
         backupentries = [x for x in backupentries if keep(x)]
 
     _playback(
         file,
         report,
         opener,
         vfsmap,
         entries,
         backupentries,
         checkambigfiles=checkambigfiles,
     )

tests/test-transaction-rollback-on-revlog-split.t

@@ -1,444 +1,448 b''
 Test correctness of revlog inline -> non-inline transition
 ----------------------------------------------------------
 
+We test various file length and naming pattern as this created issue in the
+past.
+
 Helper extension to intercept renames and kill process
 
   $ cat > $TESTTMP/intercept_before_rename.py << EOF
   > import os
   > import signal
   > from mercurial import extensions, util
   > 
   > def extsetup(ui):
   >     def rename(orig, src, dest, *args, **kwargs):
   >         path = util.normpath(dest)
   >         if path.endswith(b'data/file.i'):
   >             os.kill(os.getpid(), signal.SIGKILL)
   >         return orig(src, dest, *args, **kwargs)
   >     extensions.wrapfunction(util, 'rename', rename)
   > EOF
 
   $ cat > $TESTTMP/intercept_after_rename.py << EOF
   > import os
   > import signal
   > from mercurial import extensions, util
   > 
   > def extsetup(ui):
   >     def close(orig, *args, **kwargs):
   >         path = util.normpath(args[0]._atomictempfile__name)
   >         r = orig(*args, **kwargs)
   >         if path.endswith(b'/.hg/store/data/file.i'):
   >             os.kill(os.getpid(), signal.SIGKILL)
   >         return r
   >     extensions.wrapfunction(util.atomictempfile, 'close', close)
   > def extsetup(ui):
   >     def rename(orig, src, dest, *args, **kwargs):
   >         path = util.normpath(dest)
   >         r = orig(src, dest, *args, **kwargs)
   >         if path.endswith(b'data/file.i'):
   >             os.kill(os.getpid(), signal.SIGKILL)
   >         return r
   >     extensions.wrapfunction(util, 'rename', rename)
   > EOF
 
   $ cat > $TESTTMP/killme.py << EOF
   > import os
   > import signal
   > 
   > def killme(ui, repo, hooktype, **kwargs):
   >     os.kill(os.getpid(), signal.SIGKILL)
   > EOF
 
   $ cat > $TESTTMP/reader_wait_split.py << EOF
   > import os
   > import signal
   > from mercurial import extensions, revlog, testing
   > def _wait_post_load(orig, self, *args, **kwargs):
   >     wait = b'data/file' in self.radix
   >     if wait:
   >         testing.wait_file(b"$TESTTMP/writer-revlog-split")
   >     r = orig(self, *args, **kwargs)
   >     if wait:
   >         testing.write_file(b"$TESTTMP/reader-index-read")
   >         testing.wait_file(b"$TESTTMP/writer-revlog-unsplit")
   >     return r
   > 
   > def extsetup(ui):
   >     extensions.wrapfunction(revlog.revlog, '_loadindex', _wait_post_load)
   > EOF
 
 setup a repository for tests
 ----------------------------
 
   $ cat >> $HGRCPATH << EOF
   > [format]
   > revlog-compression=none
   > EOF
 
   $ hg init troffset-computation
   $ cd troffset-computation
   $ files="
   > file
   > Directory_With,Special%Char/Complex_File.babar
   > foo/bar/babar_celeste/foo
+  > 1234567890/1234567890/1234567890/1234567890/1234567890/1234567890/1234567890/1234567890/1234567890/1234567890/f
   > "
   $ for f in $files; do
   >     mkdir -p `dirname $f`
   > done
   $ for f in $files; do
   >     printf '%20d' '1' > $f
   > done
   $ hg commit -Aqma
   $ for f in $files; do
   >     printf '%1024d' '1' > $f
   > done
   $ hg commit -Aqmb
   $ for f in $files; do
   >     printf '%20d' '1' > $f
   > done
   $ hg commit -Aqmc
   $ for f in $files; do
   >     dd if=/dev/zero of=$f bs=1k count=128 > /dev/null 2>&1
   > done
   $ hg commit -AqmD --traceback
 
 Reference size:
   $ f -s file
   file: size=131072
   $ f -s .hg/store/data/file*
   .hg/store/data/file.d: size=132139
   .hg/store/data/file.i: size=256
 
   $ cd ..
 
 
 Test a hard crash after the file was split but before the transaction was committed
 ===================================================================================
 
 Test offset computation to correctly factor in the index entries themselves.
 Also test that the new data size has the correct size if the transaction is aborted
 after the index has been replaced.
 
 Test repo has commits a, b, c, D, where D is large (grows the revlog enough that it
 transitions to non-inline storage). The clone initially has changes a, b
 and will transition to non-inline storage when adding c, D.
 
 If the transaction adding c, D is rolled back, then we don't undo the revlog split,
 but truncate the index and the data to remove both c and D.
 
 
   $ hg clone --quiet --rev 1 troffset-computation troffset-computation-copy
   $ cd troffset-computation-copy
 
 Reference size:
   $ f -s file
   file: size=1024
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
 
   $ cat > .hg/hgrc <<EOF
   > [hooks]
   > pretxnchangegroup = python:$TESTTMP/killme.py:killme
   > EOF
 #if chg
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   [255]
 #else
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   *Killed* (glob)
   [137]
 #endif
 
 
 The inline revlog still exist, but a split version exist next to it
 
   $ f -s .hg/store/data/file*
   .hg/store/data/file.d: size=132139
   .hg/store/data/file.i: size=132395
   .hg/store/data/file.i.s: size=256
 
 
 The first file.i entry should match the "Reference size" above.
 The first file.d entry is the temporary record during the split,
 
 A "temporary file" entry exist for the split index.
 
   $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file
   data/file.i 1174
   data/file.d 0
   $ cat .hg/store/journal.backupfiles | tr -s '\000' ' ' | tr -s '\00' ' '| grep data/file
    data/file.i data/journal.backup.file.i 0
    data/file.i.s 0
 
 recover is rolling the split back, the fncache is still valid
 
   $ hg recover
   rolling back interrupted transaction
   (verify step skipped, run `hg verify` to check your repository content)
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
   $ hg tip
-  changeset:   1:272bd31be9b8
+  changeset:   1:cc8dfb126534
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     b
   
   $ hg verify -q
   $ hg debugrebuildfncache --only-data
   fncache already up to date
   $ hg verify -q
   $ cd ..
 
 Test a hard crash right before the index is move into place
 ===========================================================
 
 Now retry the procedure but intercept the rename of the index and check that
 the journal does not contain the new index size. This demonstrates the edge case
 where the data file is left as garbage.
 
   $ hg clone --quiet --rev 1 troffset-computation troffset-computation-copy2
   $ cd troffset-computation-copy2
 
 Reference size:
   $ f -s file
   file: size=1024
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
 
   $ cat > .hg/hgrc <<EOF
   > [extensions]
   > intercept_rename = $TESTTMP/intercept_before_rename.py
   > EOF
 #if chg
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   [255]
 #else
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   *Killed* (glob)
   [137]
 #endif
 
 The inline revlog still exist, but a split version exist next to it
 
   $ f -s .hg/store/data/file*
   .hg/store/data/file.d: size=132139
   .hg/store/data/file.i: size=132395
   .hg/store/data/file.i.s: size=256
 
   $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file
   data/file.i 1174
   data/file.d 0
 
 recover is rolling the split back, the fncache is still valid
 
   $ hg recover
   rolling back interrupted transaction
   (verify step skipped, run `hg verify` to check your repository content)
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
   $ hg tip
-  changeset:   1:272bd31be9b8
+  changeset:   1:cc8dfb126534
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     b
   
   $ hg verify -q
   $ cd ..
 
 Test a hard crash right after the index is move into place
 ===========================================================
 
 Now retry the procedure but intercept the rename of the index.
 
   $ hg clone --quiet --rev 1 troffset-computation troffset-computation-crash-after-rename
   $ cd troffset-computation-crash-after-rename
 
 Reference size:
   $ f -s file
   file: size=1024
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
 
   $ cat > .hg/hgrc <<EOF
   > [extensions]
   > intercept_rename = $TESTTMP/intercept_after_rename.py
   > EOF
 #if chg
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   [255]
 #else
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   *Killed* (glob)
   [137]
 #endif
 
 The inline revlog was over written on disk
 
   $ f -s .hg/store/data/file*
   .hg/store/data/file.d: size=132139
   .hg/store/data/file.i: size=256
 
   $ cat .hg/store/journal | tr -s '\000' ' ' | grep data/file
   data/file.i 1174
   data/file.d 0
 
 recover is rolling the split back, the fncache is still valid
 
   $ hg recover
   rolling back interrupted transaction
   (verify step skipped, run `hg verify` to check your repository content)
   $ f -s .hg/store/data/file*
   .hg/store/data/file.i: size=1174
   $ hg tip
-  changeset:   1:272bd31be9b8
+  changeset:   1:cc8dfb126534
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     b
   
   $ hg verify -q
   $ cd ..
 
 Have the transaction rollback itself without any hard crash
 ===========================================================
 
 
 Repeat the original test but let hg rollback the transaction.
 
   $ hg clone --quiet --rev 1 troffset-computation troffset-computation-copy-rb
   $ cd troffset-computation-copy-rb
   $ cat > .hg/hgrc <<EOF
   > [hooks]
   > pretxnchangegroup = false
   > EOF
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   transaction abort!
   rollback completed
   abort: pretxnchangegroup hook exited with status 1
   [40]
 
 The split was rollback
 
   $ f -s .hg/store/data/file*
   .hg/store/data/file.d: size=0
   .hg/store/data/file.i: size=1174
 
 
   $ hg tip
-  changeset:   1:272bd31be9b8
+  changeset:   1:cc8dfb126534
   tag:         tip
   user:        test
   date:        Thu Jan 01 00:00:00 1970 +0000
   summary:     b
   
   $ hg verify -q
   $ cd ..
 
 Read race
 =========
 
 We check that a client that started reading a revlog (its index) after the
 split and end reading (the data) after the rollback should be fine
 
   $ hg clone --quiet --rev 1 troffset-computation troffset-computation-race
   $ cd troffset-computation-race
   $ cat > .hg/hgrc <<EOF
   > [hooks]
   > pretxnchangegroup=$RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/reader-index-read $TESTTMP/writer-revlog-split
   > pretxnclose = false
   > EOF
 
 start a reader
 
   $ hg cat --rev 0 file \
   > --config "extensions.wait_read=$TESTTMP/reader_wait_split.py" \
   > 2> $TESTTMP/reader.stderr \
   > > $TESTTMP/reader.stdout &
 
 Do a failed pull in //
 
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   transaction abort!
   rollback completed
   abort: pretxnclose hook exited with status 1
   [40]
   $ touch $TESTTMP/writer-revlog-unsplit
   $ wait
 
 The reader should be fine
   $ cat $TESTTMP/reader.stderr
   $ cat $TESTTMP/reader.stdout
                      1 (no-eol)
   $ cd ..
 
 pending hooks
 =============
 
 We checks that hooks properly see the inside of the transaction, while other process don't.
 
   $ hg clone --quiet --rev 1 troffset-computation troffset-computation-hooks
   $ cd troffset-computation-hooks
   $ cat > .hg/hgrc <<EOF
   > [hooks]
   > pretxnclose.01-echo = hg cat -r 'max(all())' file | f --size
   > pretxnclose.02-echo = $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/hook-done $TESTTMP/hook-tr-ready
   > pretxnclose.03-abort = false
   > EOF
 
   $ (
   >   $RUNTESTDIR/testlib/wait-on-file 5 $TESTTMP/hook-tr-ready;\
   >   hg cat -r 'max(all())' file | f --size;\
   >   touch $TESTTMP/hook-done
   > ) >stdout 2>stderr &
 
   $ hg pull ../troffset-computation
   pulling from ../troffset-computation
   searching for changes
   adding changesets
   adding manifests
   adding file changes
   size=131072
   transaction abort!
   rollback completed
   abort: pretxnclose.03-abort hook exited with status 1
   [40]
 
   $ cat stdout
   size=1024
   $ cat stderr
 
 
   $ cd ..