sidedata: test we can successfully write sidedata

For now we just write them and rejoice on the lack of crashes.

Differential Revision: https://phab.mercurial-scm.org/D6896

tests/testlib/ext-sidedata.py

@@ -0,0 +1,36 b''
+# ext-sidedata.py - small extension to test the sidedata logic
+#
+# Copyright 2019 Pierre-Yves David <[e-mail])
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2 or any later version.
+
+from __future__ import absolute_import
+
+import hashlib
+import struct
+
+from mercurial import (
+    extensions,
+    revlog,
+)
+
+from mercurial.revlogutils import (
+    sidedata,
+)
+
+def wrapaddrevision(orig, self, text, transaction, link, p1, p2, *args,
+                    **kwargs):
+    if kwargs.get('sidedata') is None:
+        kwargs['sidedata'] = {}
+    sd = kwargs['sidedata']
+    ## let's store some arbitrary data just for testing
+    # text length
+    sd[sidedata.SD_TEST1] = struct.pack('>I', len(text))
+    # and sha2 hashes
+    sha256 = hashlib.sha256(text).digest()
+    sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256)
+    return orig(self, text, transaction, link, p1, p2, *args, **kwargs)
+
+def extsetup(ui):
+    extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision)

mercurial/revlogutils/sidedata.py

@@ -1,85 +1,96 b''
 # sidedata.py - Logic around store extra data alongside revlog revisions
 #
 # Copyright 2019 Pierre-Yves David <[e-mail])
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 """core code for "sidedata" support
 
 The "sidedata" are stored alongside the revision without actually being part of
 its content and not affecting its hash. It's main use cases is to cache
 important information related to a changesets.
 
 The current implementation is experimental and subject to changes. Do not rely
 on it in production.
 
 Sidedata are stored in the revlog itself, withing the revision rawtext. They
 are inserted, removed from it using the flagprocessors mechanism. The following
 format is currently used::
 
     initial header:
         <number of sidedata; 2 bytes>
     sidedata (repeated N times):
         <sidedata-key; 2 bytes>
         <sidedata-entry-length: 4 bytes>
         <sidedata-content-sha1-digest: 20 bytes>
         <sidedata-content; X bytes>
     normal raw text:
         <all bytes remaining in the rawtext>
 
 This is a simple and effective format. It should be enought to experiment with
 the concept.
 """
 
 from __future__ import absolute_import
 
 import hashlib
 import struct
 
 from .. import error
 
+## sidedata type constant
+# reserve a block for testing purposes.
+SD_TEST1 = 1
+SD_TEST2 = 2
+SD_TEST3 = 3
+SD_TEST4 = 4
+SD_TEST5 = 5
+SD_TEST6 = 6
+SD_TEST7 = 7
+
+# internal format constant
 SIDEDATA_HEADER = struct.Struct('>H')
 SIDEDATA_ENTRY = struct.Struct('>HL20s')
 
 def sidedatawriteprocessor(rl, text, sidedata):
     sidedata = list(sidedata.items())
     sidedata.sort()
     rawtext = [SIDEDATA_HEADER.pack(len(sidedata))]
     for key, value in sidedata:
         digest = hashlib.sha1(value).digest()
         rawtext.append(SIDEDATA_ENTRY.pack(key, len(value), digest))
     for key, value in sidedata:
         rawtext.append(value)
     rawtext.append(bytes(text))
     return ''.join(rawtext), False
 
 def sidedatareadprocessor(rl, text):
     sidedata = {}
     offset = 0
     nbentry, = SIDEDATA_HEADER.unpack(text[:SIDEDATA_HEADER.size])
     offset += SIDEDATA_HEADER.size
     dataoffset = SIDEDATA_HEADER.size + (SIDEDATA_ENTRY.size * nbentry)
     for i in range(nbentry):
         nextoffset = offset + SIDEDATA_ENTRY.size
         key, size, storeddigest = SIDEDATA_ENTRY.unpack(text[offset:nextoffset])
         offset = nextoffset
         # read the data associated with that entry
         nextdataoffset = dataoffset + size
         entrytext = text[dataoffset:nextdataoffset]
         readdigest = hashlib.sha1(entrytext).digest()
         if storeddigest != readdigest:
             raise error.SidedataHashError(key, storeddigest, readdigest)
         sidedata[key] = entrytext
         dataoffset = nextdataoffset
     text = text[dataoffset:]
     return text, True, sidedata
 
 def sidedatarawprocessor(rl, text):
     # side data modifies rawtext and prevent rawtext hash validation
     return False
 
 processors = (
     sidedatareadprocessor,
     sidedatawriteprocessor,
     sidedatarawprocessor,
 )

tests/test-sidedata.t

@@ -1,65 +1,83 b''
 ==========================================================
 Test file dedicated to checking side-data related behavior
 ==========================================================
 
+Check data can be written/read from sidedata
+============================================
+
+  $ cat << EOF >> $HGRCPATH
+  > [extensions]
+  > testsidedata=$TESTDIR/testlib/ext-sidedata.py
+  > EOF
+
+  $ hg init test-sidedata --config format.use-side-data=yes
+  $ cd test-sidedata
+  $ echo aaa > a
+  $ hg add a
+  $ hg commit -m a --traceback
+  $ echo aaa > b
+  $ hg add b
+  $ hg commit -m b
+  $ echo xxx >> a
+  $ hg commit -m aa
 
 Check upgrade behavior
 ======================
 
 Right now, sidedata has not upgrade support
 
 Check that we cannot upgrade to sidedata
 ----------------------------------------
 
   $ hg init up-no-side-data --config format.use-side-data=no
   $ hg debugformat -v -R up-no-side-data
   format-variant    repo config default
   fncache:           yes    yes     yes
   dotencode:         yes    yes     yes
   generaldelta:      yes    yes     yes
   sparserevlog:      yes    yes     yes
   sidedata:           no     no      no
   plain-cl-delta:    yes    yes     yes
   compression:       zlib   zlib    zlib
   compression-level: default default default
   $ hg debugformat -v -R up-no-side-data --config format.use-side-data=yes
   format-variant    repo config default
   fncache:           yes    yes     yes
   dotencode:         yes    yes     yes
   generaldelta:      yes    yes     yes
   sparserevlog:      yes    yes     yes
   sidedata:           no    yes      no
   plain-cl-delta:    yes    yes     yes
   compression:       zlib   zlib    zlib
   compression-level: default default default
   $ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes
   abort: cannot upgrade repository; do not support adding requirement: exp-sidedata-flag
   [255]
 
 Check that we cannot upgrade to sidedata
 ----------------------------------------
 
   $ hg init up-side-data --config format.use-side-data=yes
   $ hg debugformat -v -R up-side-data
   format-variant    repo config default
   fncache:           yes    yes     yes
   dotencode:         yes    yes     yes
   generaldelta:      yes    yes     yes
   sparserevlog:      yes    yes     yes
   sidedata:          yes     no      no
   plain-cl-delta:    yes    yes     yes
   compression:       zlib   zlib    zlib
   compression-level: default default default
   $ hg debugformat -v -R up-side-data --config format.use-side-data=no
   format-variant    repo config default
   fncache:           yes    yes     yes
   dotencode:         yes    yes     yes
   generaldelta:      yes    yes     yes
   sparserevlog:      yes    yes     yes
   sidedata:          yes     no      no
   plain-cl-delta:    yes    yes     yes
   compression:       zlib   zlib    zlib
   compression-level: default default default
   $ hg debugupgraderepo -R up-side-data --config format.use-side-data=no
   abort: cannot upgrade repository; requirement would be removed: exp-sidedata-flag
   [255]