commitctx: extract copy information encoding into extra into commit.py

The encoding of copy information into extra has multiple subcases and become
quite complicated (eg: empty list can be explicitly or implicitly stored for
example). In addition, it is niche experimental feature since as it affect the
hash, it is only suitable for user who don't mercurial for storage server side
(ie: Google).

Having this complexity part of the changelog will get in the way of further
cleanup. We could have to either move more of that logic into the changelog or
to move or extract more of the logic at the higher level. We take the second
approach and start gather logic in dedicated function in commit.py.

mercurial/changelog.py

@@ -1,627 +1,605
 # changelog.py - changelog class for mercurial
 #
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 from .i18n import _
 from .node import (
     bin,
     hex,
     nullid,
 )
 from .thirdparty import attr
 
 from . import (
     encoding,
     error,
     metadata,
     pycompat,
     revlog,
 )
 from .utils import (
     dateutil,
     stringutil,
 )
 
 from .revlogutils import sidedata as sidedatamod
 
 _defaultextra = {b'branch': b'default'}
 
 
 def _string_escape(text):
     """
     >>> from .pycompat import bytechr as chr
     >>> d = {b'nl': chr(10), b'bs': chr(92), b'cr': chr(13), b'nul': chr(0)}
     >>> s = b"ab%(nl)scd%(bs)s%(bs)sn%(nul)s12ab%(cr)scd%(bs)s%(nl)s" % d
     >>> s
     'ab\\ncd\\\\\\\\n\\x0012ab\\rcd\\\\\\n'
     >>> res = _string_escape(s)
     >>> s == _string_unescape(res)
     True
     """
     # subset of the string_escape codec
     text = (
         text.replace(b'\\', b'\\\\')
         .replace(b'\n', b'\\n')
         .replace(b'\r', b'\\r')
     )
     return text.replace(b'\0', b'\\0')
 
 
 def _string_unescape(text):
     if b'\\0' in text:
         # fix up \0 without getting into trouble with \\0
         text = text.replace(b'\\\\', b'\\\\\n')
         text = text.replace(b'\\0', b'\0')
         text = text.replace(b'\n', b'')
     return stringutil.unescapestr(text)
 
 
 def decodeextra(text):
     """
     >>> from .pycompat import bytechr as chr
     >>> sorted(decodeextra(encodeextra({b'foo': b'bar', b'baz': chr(0) + b'2'})
     ...                    ).items())
     [('baz', '\\x002'), ('branch', 'default'), ('foo', 'bar')]
     >>> sorted(decodeextra(encodeextra({b'foo': b'bar',
     ...                                 b'baz': chr(92) + chr(0) + b'2'})
     ...                    ).items())
     [('baz', '\\\\\\x002'), ('branch', 'default'), ('foo', 'bar')]
     """
     extra = _defaultextra.copy()
     for l in text.split(b'\0'):
         if l:
             k, v = _string_unescape(l).split(b':', 1)
             extra[k] = v
     return extra
 
 
 def encodeextra(d):
     # keys must be sorted to produce a deterministic changelog entry
     items = [_string_escape(b'%s:%s' % (k, d[k])) for k in sorted(d)]
     return b"\0".join(items)
 
 
 def stripdesc(desc):
     """strip trailing whitespace and leading and trailing empty lines"""
     return b'\n'.join([l.rstrip() for l in desc.splitlines()]).strip(b'\n')
 
 
 class appender(object):
     '''the changelog index must be updated last on disk, so we use this class
     to delay writes to it'''
 
     def __init__(self, vfs, name, mode, buf):
         self.data = buf
         fp = vfs(name, mode)
         self.fp = fp
         self.offset = fp.tell()
         self.size = vfs.fstat(fp).st_size
         self._end = self.size
 
     def end(self):
         return self._end
 
     def tell(self):
         return self.offset
 
     def flush(self):
         pass
 
     @property
     def closed(self):
         return self.fp.closed
 
     def close(self):
         self.fp.close()
 
     def seek(self, offset, whence=0):
         '''virtual file offset spans real file and data'''
         if whence == 0:
             self.offset = offset
         elif whence == 1:
             self.offset += offset
         elif whence == 2:
             self.offset = self.end() + offset
         if self.offset < self.size:
             self.fp.seek(self.offset)
 
     def read(self, count=-1):
         '''only trick here is reads that span real file and data'''
         ret = b""
         if self.offset < self.size:
             s = self.fp.read(count)
             ret = s
             self.offset += len(s)
             if count > 0:
                 count -= len(s)
         if count != 0:
             doff = self.offset - self.size
             self.data.insert(0, b"".join(self.data))
             del self.data[1:]
             s = self.data[0][doff : doff + count]
             self.offset += len(s)
             ret += s
         return ret
 
     def write(self, s):
         self.data.append(bytes(s))
         self.offset += len(s)
         self._end += len(s)
 
     def __enter__(self):
         self.fp.__enter__()
         return self
 
     def __exit__(self, *args):
         return self.fp.__exit__(*args)
 
 
 class _divertopener(object):
     def __init__(self, opener, target):
         self._opener = opener
         self._target = target
 
     def __call__(self, name, mode=b'r', checkambig=False, **kwargs):
         if name != self._target:
             return self._opener(name, mode, **kwargs)
         return self._opener(name + b".a", mode, **kwargs)
 
     def __getattr__(self, attr):
         return getattr(self._opener, attr)
 
 
 def _delayopener(opener, target, buf):
     """build an opener that stores chunks in 'buf' instead of 'target'"""
 
     def _delay(name, mode=b'r', checkambig=False, **kwargs):
         if name != target:
             return opener(name, mode, **kwargs)
         assert not kwargs
         return appender(opener, name, mode, buf)
 
     return _delay
 
 
 @attr.s
 class _changelogrevision(object):
     # Extensions might modify _defaultextra, so let the constructor below pass
     # it in
     extra = attr.ib()
     manifest = attr.ib(default=nullid)
     user = attr.ib(default=b'')
     date = attr.ib(default=(0, 0))
     files = attr.ib(default=attr.Factory(list))
     filesadded = attr.ib(default=None)
     filesremoved = attr.ib(default=None)
     p1copies = attr.ib(default=None)
     p2copies = attr.ib(default=None)
     description = attr.ib(default=b'')
 
 
 class changelogrevision(object):
     """Holds results of a parsed changelog revision.
 
     Changelog revisions consist of multiple pieces of data, including
     the manifest node, user, and date. This object exposes a view into
     the parsed object.
     """
 
     __slots__ = (
         '_offsets',
         '_text',
         '_sidedata',
         '_cpsd',
     )
 
     def __new__(cls, text, sidedata, cpsd):
         if not text:
             return _changelogrevision(extra=_defaultextra)
 
         self = super(changelogrevision, cls).__new__(cls)
         # We could return here and implement the following as an __init__.
         # But doing it here is equivalent and saves an extra function call.
 
         # format used:
         # nodeid\n        : manifest node in ascii
         # user\n          : user, no \n or \r allowed
         # time tz extra\n : date (time is int or float, timezone is int)
         #                 : extra is metadata, encoded and separated by '\0'
         #                 : older versions ignore it
         # files\n\n       : files modified by the cset, no \n or \r allowed
         # (.*)            : comment (free text, ideally utf-8)
         #
         # changelog v0 doesn't use extra
 
         nl1 = text.index(b'\n')
         nl2 = text.index(b'\n', nl1 + 1)
         nl3 = text.index(b'\n', nl2 + 1)
 
         # The list of files may be empty. Which means nl3 is the first of the
         # double newline that precedes the description.
         if text[nl3 + 1 : nl3 + 2] == b'\n':
             doublenl = nl3
         else:
             doublenl = text.index(b'\n\n', nl3 + 1)
 
         self._offsets = (nl1, nl2, nl3, doublenl)
         self._text = text
         self._sidedata = sidedata
         self._cpsd = cpsd
 
         return self
 
     @property
     def manifest(self):
         return bin(self._text[0 : self._offsets[0]])
 
     @property
     def user(self):
         off = self._offsets
         return encoding.tolocal(self._text[off[0] + 1 : off[1]])
 
     @property
     def _rawdate(self):
         off = self._offsets
         dateextra = self._text[off[1] + 1 : off[2]]
         return dateextra.split(b' ', 2)[0:2]
 
     @property
     def _rawextra(self):
         off = self._offsets
         dateextra = self._text[off[1] + 1 : off[2]]
         fields = dateextra.split(b' ', 2)
         if len(fields) != 3:
             return None
 
         return fields[2]
 
     @property
     def date(self):
         raw = self._rawdate
         time = float(raw[0])
         # Various tools did silly things with the timezone.
         try:
             timezone = int(raw[1])
         except ValueError:
             timezone = 0
 
         return time, timezone
 
     @property
     def extra(self):
         raw = self._rawextra
         if raw is None:
             return _defaultextra
 
         return decodeextra(raw)
 
     @property
     def files(self):
         off = self._offsets
         if off[2] == off[3]:
             return []
 
         return self._text[off[2] + 1 : off[3]].split(b'\n')
 
     @property
     def filesadded(self):
         if self._cpsd:
             rawindices = self._sidedata.get(sidedatamod.SD_FILESADDED)
             if not rawindices:
                 return []
         else:
             rawindices = self.extra.get(b'filesadded')
         if rawindices is None:
             return None
         return metadata.decodefileindices(self.files, rawindices)
 
     @property
     def filesremoved(self):
         if self._cpsd:
             rawindices = self._sidedata.get(sidedatamod.SD_FILESREMOVED)
             if not rawindices:
                 return []
         else:
             rawindices = self.extra.get(b'filesremoved')
         if rawindices is None:
             return None
         return metadata.decodefileindices(self.files, rawindices)
 
     @property
     def p1copies(self):
         if self._cpsd:
             rawcopies = self._sidedata.get(sidedatamod.SD_P1COPIES)
             if not rawcopies:
                 return {}
         else:
             rawcopies = self.extra.get(b'p1copies')
         if rawcopies is None:
             return None
         return metadata.decodecopies(self.files, rawcopies)
 
     @property
     def p2copies(self):
         if self._cpsd:
             rawcopies = self._sidedata.get(sidedatamod.SD_P2COPIES)
             if not rawcopies:
                 return {}
         else:
             rawcopies = self.extra.get(b'p2copies')
         if rawcopies is None:
             return None
         return metadata.decodecopies(self.files, rawcopies)
 
     @property
     def description(self):
         return encoding.tolocal(self._text[self._offsets[3] + 2 :])
 
 
 class changelog(revlog.revlog):
     def __init__(self, opener, trypending=False):
         """Load a changelog revlog using an opener.
 
         If ``trypending`` is true, we attempt to load the index from a
         ``00changelog.i.a`` file instead of the default ``00changelog.i``.
         The ``00changelog.i.a`` file contains index (and possibly inline
         revision) data for a transaction that hasn't been finalized yet.
         It exists in a separate file to facilitate readers (such as
         hooks processes) accessing data before a transaction is finalized.
         """
         if trypending and opener.exists(b'00changelog.i.a'):
             indexfile = b'00changelog.i.a'
         else:
             indexfile = b'00changelog.i'
 
         datafile = b'00changelog.d'
         revlog.revlog.__init__(
             self,
             opener,
             indexfile,
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
             persistentnodemap=opener.options.get(b'persistent-nodemap', False),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):
             # changelogs don't benefit from generaldelta.
 
             self.version &= ~revlog.FLAG_GENERALDELTA
             self._generaldelta = False
 
         # Delta chains for changelogs tend to be very small because entries
         # tend to be small and don't delta well with each. So disable delta
         # chains.
         self._storedeltachains = False
 
         self._realopener = opener
         self._delayed = False
         self._delaybuf = None
         self._divert = False
         self.filteredrevs = frozenset()
         self._copiesstorage = opener.options.get(b'copies-storage')
 
     def delayupdate(self, tr):
         """delay visibility of index updates to other readers"""
 
         if not self._delayed:
             if len(self) == 0:
                 self._divert = True
                 if self._realopener.exists(self.indexfile + b'.a'):
                     self._realopener.unlink(self.indexfile + b'.a')
                 self.opener = _divertopener(self._realopener, self.indexfile)
             else:
                 self._delaybuf = []
                 self.opener = _delayopener(
                     self._realopener, self.indexfile, self._delaybuf
                 )
         self._delayed = True
         tr.addpending(b'cl-%i' % id(self), self._writepending)
         tr.addfinalize(b'cl-%i' % id(self), self._finalize)
 
     def _finalize(self, tr):
         """finalize index updates"""
         self._delayed = False
         self.opener = self._realopener
         # move redirected index data back into place
         if self._divert:
             assert not self._delaybuf
             tmpname = self.indexfile + b".a"
             nfile = self.opener.open(tmpname)
             nfile.close()
             self.opener.rename(tmpname, self.indexfile, checkambig=True)
         elif self._delaybuf:
             fp = self.opener(self.indexfile, b'a', checkambig=True)
             fp.write(b"".join(self._delaybuf))
             fp.close()
             self._delaybuf = None
         self._divert = False
         # split when we're done
         self._enforceinlinesize(tr)
 
     def _writepending(self, tr):
         """create a file containing the unfinalized state for
         pretxnchangegroup"""
         if self._delaybuf:
             # make a temporary copy of the index
             fp1 = self._realopener(self.indexfile)
             pendingfilename = self.indexfile + b".a"
             # register as a temp file to ensure cleanup on failure
             tr.registertmp(pendingfilename)
             # write existing data
             fp2 = self._realopener(pendingfilename, b"w")
             fp2.write(fp1.read())
             # add pending data
             fp2.write(b"".join(self._delaybuf))
             fp2.close()
             # switch modes so finalize can simply rename
             self._delaybuf = None
             self._divert = True
             self.opener = _divertopener(self._realopener, self.indexfile)
 
         if self._divert:
             return True
 
         return False
 
     def _enforceinlinesize(self, tr, fp=None):
         if not self._delayed:
             revlog.revlog._enforceinlinesize(self, tr, fp)
 
     def read(self, node):
         """Obtain data from a parsed changelog revision.
 
         Returns a 6-tuple of:
 
            - manifest node in binary
            - author/user as a localstr
            - date as a 2-tuple of (time, timezone)
            - list of files
            - commit message as a localstr
            - dict of extra metadata
 
         Unless you need to access all fields, consider calling
         ``changelogrevision`` instead, as it is faster for partial object
         access.
         """
         d, s = self._revisiondata(node)
         c = changelogrevision(
             d, s, self._copiesstorage == b'changeset-sidedata'
         )
         return (c.manifest, c.user, c.date, c.files, c.description, c.extra)
 
     def changelogrevision(self, nodeorrev):
         """Obtain a ``changelogrevision`` for a node or revision."""
         text, sidedata = self._revisiondata(nodeorrev)
         return changelogrevision(
             text, sidedata, self._copiesstorage == b'changeset-sidedata'
         )
 
     def readfiles(self, node):
         """
         short version of read that only returns the files modified by the cset
         """
         text = self.revision(node)
         if not text:
             return []
         last = text.index(b"\n\n")
         l = text[:last].split(b'\n')
         return l[3:]
 
     def add(
         self,
         manifest,
         files,
         desc,
         transaction,
         p1,
         p2,
         user,
         date=None,
         extra=None,
         p1copies=None,
         p2copies=None,
         filesadded=None,
         filesremoved=None,
     ):
         # Convert to UTF-8 encoded bytestrings as the very first
         # thing: calling any method on a localstr object will turn it
         # into a str object and the cached UTF-8 string is thus lost.
         user, desc = encoding.fromlocal(user), encoding.fromlocal(desc)
 
         user = user.strip()
         # An empty username or a username with a "\n" will make the
         # revision text contain two "\n\n" sequences -> corrupt
         # repository since read cannot unpack the revision.
         if not user:
             raise error.StorageError(_(b"empty username"))
         if b"\n" in user:
             raise error.StorageError(
                 _(b"username %r contains a newline") % pycompat.bytestr(user)
             )
 
         desc = stripdesc(desc)
 
         if date:
             parseddate = b"%d %d" % dateutil.parsedate(date)
         else:
             parseddate = b"%d %d" % dateutil.makedate()
         if extra:
             branch = extra.get(b"branch")
             if branch in (b"default", b""):
                 del extra[b"branch"]
             elif branch in (b".", b"null", b"tip"):
                 raise error.StorageError(
                     _(b'the name \'%s\' is reserved') % branch
                 )
         sortedfiles = sorted(files)
         sidedata = None
-        if extra is not None:
-            for name in (
-                b'p1copies',
-                b'p2copies',
-                b'filesadded',
-                b'filesremoved',
-            ):
-                extra.pop(name, None)
-        if p1copies is not None:
-            p1copies = metadata.encodecopies(sortedfiles, p1copies)
-        if p2copies is not None:
-            p2copies = metadata.encodecopies(sortedfiles, p2copies)
-        if filesadded is not None:
-            filesadded = metadata.encodefileindices(sortedfiles, filesadded)
-        if filesremoved is not None:
-            filesremoved = metadata.encodefileindices(sortedfiles, filesremoved)
-        if self._copiesstorage == b'extra':
-            extrasentries = p1copies, p2copies, filesadded, filesremoved
-            if extra is None and any(x is not None for x in extrasentries):
-                extra = {}
-            if p1copies is not None:
-                extra[b'p1copies'] = p1copies
-            if p2copies is not None:
-                extra[b'p2copies'] = p2copies
-            if filesadded is not None:
-                extra[b'filesadded'] = filesadded
-            if filesremoved is not None:
-                extra[b'filesremoved'] = filesremoved
-        elif self._copiesstorage == b'changeset-sidedata':
+        if self._copiesstorage == b'changeset-sidedata':
             sidedata = {}
             if p1copies:
+                p1copies = metadata.encodecopies(sortedfiles, p1copies)
                 sidedata[sidedatamod.SD_P1COPIES] = p1copies
             if p2copies:
+                p2copies = metadata.encodecopies(sortedfiles, p2copies)
                 sidedata[sidedatamod.SD_P2COPIES] = p2copies
             if filesadded:
+                filesadded = metadata.encodefileindices(sortedfiles, filesadded)
                 sidedata[sidedatamod.SD_FILESADDED] = filesadded
             if filesremoved:
+                filesremoved = metadata.encodefileindices(
+                    sortedfiles, filesremoved
+                )
                 sidedata[sidedatamod.SD_FILESREMOVED] = filesremoved
             if not sidedata:
                 sidedata = None
 
         if extra:
             extra = encodeextra(extra)
             parseddate = b"%s %s" % (parseddate, extra)
         l = [hex(manifest), user, parseddate] + sortedfiles + [b"", desc]
         text = b"\n".join(l)
         return self.addrevision(
             text, transaction, len(self), p1, p2, sidedata=sidedata
         )
 
     def branchinfo(self, rev):
         """return the branch name and open/close state of a revision
 
         This function exists because creating a changectx object
         just to access this is costly."""
         extra = self.read(rev)[5]
         return encoding.tolocal(extra.get(b"branch")), b'close' in extra
 
     def _nodeduplicatecallback(self, transaction, node):
         # keep track of revisions that got "re-added", eg: unbunde of know rev.
         #
         # We track them in a list to preserve their order from the source bundle
         duplicates = transaction.changes.setdefault(b'revduplicates', [])
         duplicates.append(self.rev(node))

mercurial/commit.py

@@ -1,409 +1,445
 # commit.py - fonction to perform commit
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 
 from .i18n import _
 from .node import (
     hex,
     nullid,
     nullrev,
 )
 
 from . import (
     context,
     mergestate,
     metadata,
     phases,
     scmutil,
     subrepoutil,
 )
 
 
 def _write_copy_meta(repo):
     """return a (changelog, filelog) boolean tuple
 
     changelog: copy related information should be stored in the changeset
     filelof:   copy related information should be written in the file revision
     """
     if repo.filecopiesmode == b'changeset-sidedata':
         writechangesetcopy = True
         writefilecopymeta = True
     else:
         writecopiesto = repo.ui.config(b'experimental', b'copies.write-to')
         writefilecopymeta = writecopiesto != b'changeset-only'
         writechangesetcopy = writecopiesto in (
             b'changeset-only',
             b'compatibility',
         )
     return writechangesetcopy, writefilecopymeta
 
 
 def commitctx(repo, ctx, error=False, origctx=None):
     """Add a new revision to the target repository.
     Revision information is passed via the context argument.
 
     ctx.files() should list all files involved in this commit, i.e.
     modified/added/removed files. On merge, it may be wider than the
     ctx.files() to be committed, since any file nodes derived directly
     from p1 or p2 are excluded from the committed ctx.files().
 
     origctx is for convert to work around the problem that bug
     fixes to the files list in changesets change hashes. For
     convert to be the identity, it can pass an origctx and this
     function will use the same files list when it makes sense to
     do so.
     """
     repo = repo.unfiltered()
 
     p1, p2 = ctx.p1(), ctx.p2()
     user = ctx.user()
 
     with repo.lock(), repo.transaction(b"commit") as tr:
         r = _prepare_files(tr, ctx, error=error, origctx=origctx)
         mn, files, p1copies, p2copies, filesadded, filesremoved = r
 
         extra = ctx.extra().copy()
 
+        files = sorted(files)
+        if extra is not None:
+            for name in (
+                b'p1copies',
+                b'p2copies',
+                b'filesadded',
+                b'filesremoved',
+            ):
+                extra.pop(name, None)
+        if repo.changelog._copiesstorage == b'extra':
+            extra = _extra_with_copies(
+                repo, extra, files, p1copies, p2copies, filesadded, filesremoved
+            )
+
         # update changelog
         repo.ui.note(_(b"committing changelog\n"))
         repo.changelog.delayupdate(tr)
         n = repo.changelog.add(
             mn,
             files,
             ctx.description(),
             tr,
             p1.node(),
             p2.node(),
             user,
             ctx.date(),
             extra,
             p1copies,
             p2copies,
             filesadded,
             filesremoved,
         )
         xp1, xp2 = p1.hex(), p2 and p2.hex() or b''
         repo.hook(
             b'pretxncommit', throw=True, node=hex(n), parent1=xp1, parent2=xp2,
         )
         # set the new commit is proper phase
         targetphase = subrepoutil.newcommitphase(repo.ui, ctx)
         if targetphase:
             # retract boundary do not alter parent changeset.
             # if a parent have higher the resulting phase will
             # be compliant anyway
             #
             # if minimal phase was 0 we don't need to retract anything
             phases.registernew(repo, tr, targetphase, [n])
         return n
 
 
 def _prepare_files(tr, ctx, error=False, origctx=None):
     repo = ctx.repo()
     p1 = ctx.p1()
 
     writechangesetcopy, writefilecopymeta = _write_copy_meta(repo)
 
     p1copies, p2copies = None, None
     if writechangesetcopy:
         p1copies = ctx.p1copies()
         p2copies = ctx.p2copies()
     filesadded, filesremoved = None, None
     if ctx.manifestnode():
         # reuse an existing manifest revision
         repo.ui.debug(b'reusing known manifest\n')
         mn = ctx.manifestnode()
         files = ctx.files()
         if writechangesetcopy:
             filesadded = ctx.filesadded()
             filesremoved = ctx.filesremoved()
     elif not ctx.files():
         repo.ui.debug(b'reusing manifest from p1 (no file change)\n')
         mn = p1.manifestnode()
         files = []
     else:
         mn, files, added, removed = _process_files(tr, ctx, error=error)
         if writechangesetcopy:
             filesremoved = removed
             filesadded = added
 
     if origctx and origctx.manifestnode() == mn:
         files = origctx.files()
 
     if not writefilecopymeta:
         # If writing only to changeset extras, use None to indicate that
         # no entry should be written. If writing to both, write an empty
         # entry to prevent the reader from falling back to reading
         # filelogs.
         p1copies = p1copies or None
         p2copies = p2copies or None
         filesadded = filesadded or None
         filesremoved = filesremoved or None
 
     return mn, files, p1copies, p2copies, filesadded, filesremoved
 
 
 def _process_files(tr, ctx, error=False):
     repo = ctx.repo()
     p1 = ctx.p1()
     p2 = ctx.p2()
 
     writechangesetcopy, writefilecopymeta = _write_copy_meta(repo)
 
     m1ctx = p1.manifestctx()
     m2ctx = p2.manifestctx()
     mctx = m1ctx.copy()
 
     m = mctx.read()
     m1 = m1ctx.read()
     m2 = m2ctx.read()
 
     # check in files
     added = []
     filesadded = []
     removed = list(ctx.removed())
     touched = []
     linkrev = len(repo)
     repo.ui.note(_(b"committing files:\n"))
     uipathfn = scmutil.getuipathfn(repo)
     for f in sorted(ctx.modified() + ctx.added()):
         repo.ui.note(uipathfn(f) + b"\n")
         try:
             fctx = ctx[f]
             if fctx is None:
                 removed.append(f)
             else:
                 added.append(f)
                 m[f], is_touched = _filecommit(
                     repo, fctx, m1, m2, linkrev, tr, writefilecopymeta,
                 )
                 if is_touched:
                     touched.append(f)
                     if is_touched == 'added':
                         filesadded.append(f)
                 m.setflag(f, fctx.flags())
         except OSError:
             repo.ui.warn(_(b"trouble committing %s!\n") % uipathfn(f))
             raise
         except IOError as inst:
             errcode = getattr(inst, 'errno', errno.ENOENT)
             if error or errcode and errcode != errno.ENOENT:
                 repo.ui.warn(_(b"trouble committing %s!\n") % uipathfn(f))
             raise
 
     # update manifest
     removed = [f for f in removed if f in m1 or f in m2]
     drop = sorted([f for f in removed if f in m])
     for f in drop:
         del m[f]
     if p2.rev() != nullrev:
         rf = metadata.get_removal_filter(ctx, (p1, p2, m1, m2))
         removed = [f for f in removed if not rf(f)]
 
     touched.extend(removed)
 
     files = touched
     mn = _commit_manifest(tr, linkrev, ctx, mctx, m, files, added, drop)
 
     return mn, files, filesadded, removed
 
 
 def _filecommit(
     repo, fctx, manifest1, manifest2, linkrev, tr, includecopymeta,
 ):
     """
     commit an individual file as part of a larger transaction
 
     input:
 
         fctx:       a file context with the content we are trying to commit
         manifest1:  manifest of changeset first parent
         manifest2:  manifest of changeset second parent
         linkrev:    revision number of the changeset being created
         tr:         current transation
         individual: boolean, set to False to skip storing the copy data
                     (only used by the Google specific feature of using
                     changeset extra as copy source of truth).
 
     output: (filenode, touched)
 
         filenode: the filenode that should be used by this changeset
         touched:  one of: None (mean untouched), 'added' or 'modified'
     """
 
     fname = fctx.path()
     fparent1 = manifest1.get(fname, nullid)
     fparent2 = manifest2.get(fname, nullid)
     touched = None
     if fparent1 == fparent2 == nullid:
         touched = 'added'
 
     if isinstance(fctx, context.filectx):
         # This block fast path most comparisons which are usually done. It
         # assumes that bare filectx is used and no merge happened, hence no
         # need to create a new file revision in this case.
         node = fctx.filenode()
         if node in [fparent1, fparent2]:
             repo.ui.debug(b'reusing %s filelog entry\n' % fname)
             if (
                 fparent1 != nullid and manifest1.flags(fname) != fctx.flags()
             ) or (
                 fparent2 != nullid and manifest2.flags(fname) != fctx.flags()
             ):
                 touched = 'modified'
             return node, touched
 
     flog = repo.file(fname)
     meta = {}
     cfname = fctx.copysource()
     fnode = None
 
     if cfname and cfname != fname:
         # Mark the new revision of this file as a copy of another
         # file.  This copy data will effectively act as a parent
         # of this new revision.  If this is a merge, the first
         # parent will be the nullid (meaning "look up the copy data")
         # and the second one will be the other parent.  For example:
         #
         # 0 --- 1 --- 3   rev1 changes file foo
         #   \       /     rev2 renames foo to bar and changes it
         #    \- 2 -/      rev3 should have bar with all changes and
         #                      should record that bar descends from
         #                      bar in rev2 and foo in rev1
         #
         # this allows this merge to succeed:
         #
         # 0 --- 1 --- 3   rev4 reverts the content change from rev2
         #   \       /     merging rev3 and rev4 should use bar@rev2
         #    \- 2 --- 4        as the merge base
         #
 
         cnode = manifest1.get(cfname)
         newfparent = fparent2
 
         if manifest2:  # branch merge
             if fparent2 == nullid or cnode is None:  # copied on remote side
                 if cfname in manifest2:
                     cnode = manifest2[cfname]
                     newfparent = fparent1
 
         # Here, we used to search backwards through history to try to find
         # where the file copy came from if the source of a copy was not in
         # the parent directory. However, this doesn't actually make sense to
         # do (what does a copy from something not in your working copy even
         # mean?) and it causes bugs (eg, issue4476). Instead, we will warn
         # the user that copy information was dropped, so if they didn't
         # expect this outcome it can be fixed, but this is the correct
         # behavior in this circumstance.
 
         if cnode:
             repo.ui.debug(b" %s: copy %s:%s\n" % (fname, cfname, hex(cnode)))
             if includecopymeta:
                 meta[b"copy"] = cfname
                 meta[b"copyrev"] = hex(cnode)
             fparent1, fparent2 = nullid, newfparent
         else:
             repo.ui.warn(
                 _(
                     b"warning: can't find ancestor for '%s' "
                     b"copied from '%s'!\n"
                 )
                 % (fname, cfname)
             )
 
     elif fparent1 == nullid:
         fparent1, fparent2 = fparent2, nullid
     elif fparent2 != nullid:
         # is one parent an ancestor of the other?
         fparentancestors = flog.commonancestorsheads(fparent1, fparent2)
         if fparent1 in fparentancestors:
             fparent1, fparent2 = fparent2, nullid
         elif fparent2 in fparentancestors:
             fparent2 = nullid
         elif not fparentancestors:
             # TODO: this whole if-else might be simplified much more
             ms = mergestate.mergestate.read(repo)
             if (
                 fname in ms
                 and ms[fname] == mergestate.MERGE_RECORD_MERGED_OTHER
             ):
                 fparent1, fparent2 = fparent2, nullid
 
     # is the file changed?
     text = fctx.data()
     if fparent2 != nullid or meta or flog.cmp(fparent1, text):
         if touched is None:  # do not overwrite added
             touched = 'modified'
         fnode = flog.add(text, meta, tr, linkrev, fparent1, fparent2)
     # are just the flags changed during merge?
     elif fname in manifest1 and manifest1.flags(fname) != fctx.flags():
         touched = 'modified'
         fnode = fparent1
     else:
         fnode = fparent1
     return fnode, touched
 
 
 def _commit_manifest(tr, linkrev, ctx, mctx, manifest, files, added, drop):
     """make a new manifest entry (or reuse a new one)
 
     given an initialised manifest context and precomputed list of
     - files: files affected by the commit
     - added: new entries in the manifest
     - drop:  entries present in parents but absent of this one
 
     Create a new manifest revision, reuse existing ones if possible.
 
     Return the nodeid of the manifest revision.
     """
     repo = ctx.repo()
 
     md = None
 
     # all this is cached, so it is find to get them all from the ctx.
     p1 = ctx.p1()
     p2 = ctx.p2()
     m1ctx = p1.manifestctx()
 
     m1 = m1ctx.read()
 
     if not files:
         # if no "files" actually changed in terms of the changelog,
         # try hard to detect unmodified manifest entry so that the
         # exact same commit can be reproduced later on convert.
         md = m1.diff(manifest, scmutil.matchfiles(repo, ctx.files()))
     if not files and md:
         repo.ui.debug(
             b'not reusing manifest (no file change in '
             b'changelog, but manifest differs)\n'
         )
     if files or md:
         repo.ui.note(_(b"committing manifest\n"))
         # we're using narrowmatch here since it's already applied at
         # other stages (such as dirstate.walk), so we're already
         # ignoring things outside of narrowspec in most cases. The
         # one case where we might have files outside the narrowspec
         # at this point is merges, and we already error out in the
         # case where the merge has files outside of the narrowspec,
         # so this is safe.
         mn = mctx.write(
             tr,
             linkrev,
             p1.manifestnode(),
             p2.manifestnode(),
             added,
             drop,
             match=repo.narrowmatch(),
         )
     else:
         repo.ui.debug(
             b'reusing manifest from p1 (listed files ' b'actually unchanged)\n'
         )
         mn = p1.manifestnode()
 
     return mn
+
+
+def _extra_with_copies(
+    repo, extra, files, p1copies, p2copies, filesadded, filesremoved
+):
+    """encode copy information into a `extra` dictionnary"""
+    extrasentries = p1copies, p2copies, filesadded, filesremoved
+    if extra is None and any(x is not None for x in extrasentries):
+        extra = {}
+    if p1copies is not None:
+        p1copies = metadata.encodecopies(files, p1copies)
+        extra[b'p1copies'] = p1copies
+    if p2copies is not None:
+        p2copies = metadata.encodecopies(files, p2copies)
+        extra[b'p2copies'] = p2copies
+    if filesadded is not None:
+        filesadded = metadata.encodefileindices(files, filesadded)
+        extra[b'filesadded'] = filesadded
+    if filesremoved is not None:
+        filesremoved = metadata.encodefileindices(files, filesremoved)
+        extra[b'filesremoved'] = filesremoved
+    return extra