Show More
@@ -0,0 +1,36 b'' | |||||
|
1 | # ext-sidedata.py - small extension to test the sidedata logic | |||
|
2 | # | |||
|
3 | # Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net) | |||
|
4 | # | |||
|
5 | # This software may be used and distributed according to the terms of the | |||
|
6 | # GNU General Public License version 2 or any later version. | |||
|
7 | ||||
|
8 | from __future__ import absolute_import | |||
|
9 | ||||
|
10 | import hashlib | |||
|
11 | import struct | |||
|
12 | ||||
|
13 | from mercurial import ( | |||
|
14 | extensions, | |||
|
15 | revlog, | |||
|
16 | ) | |||
|
17 | ||||
|
18 | from mercurial.revlogutils import ( | |||
|
19 | sidedata, | |||
|
20 | ) | |||
|
21 | ||||
|
22 | def wrapaddrevision(orig, self, text, transaction, link, p1, p2, *args, | |||
|
23 | **kwargs): | |||
|
24 | if kwargs.get('sidedata') is None: | |||
|
25 | kwargs['sidedata'] = {} | |||
|
26 | sd = kwargs['sidedata'] | |||
|
27 | ## let's store some arbitrary data just for testing | |||
|
28 | # text length | |||
|
29 | sd[sidedata.SD_TEST1] = struct.pack('>I', len(text)) | |||
|
30 | # and sha2 hashes | |||
|
31 | sha256 = hashlib.sha256(text).digest() | |||
|
32 | sd[sidedata.SD_TEST2] = struct.pack('>32s', sha256) | |||
|
33 | return orig(self, text, transaction, link, p1, p2, *args, **kwargs) | |||
|
34 | ||||
|
35 | def extsetup(ui): | |||
|
36 | extensions.wrapfunction(revlog.revlog, 'addrevision', wrapaddrevision) |
@@ -1,85 +1,96 b'' | |||||
1 | # sidedata.py - Logic around store extra data alongside revlog revisions |
|
1 | # sidedata.py - Logic around store extra data alongside revlog revisions | |
2 | # |
|
2 | # | |
3 | # Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net) |
|
3 | # Copyright 2019 Pierre-Yves David <pierre-yves.david@octobus.net) | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 | """core code for "sidedata" support |
|
7 | """core code for "sidedata" support | |
8 |
|
8 | |||
9 | The "sidedata" are stored alongside the revision without actually being part of |
|
9 | The "sidedata" are stored alongside the revision without actually being part of | |
10 | its content and not affecting its hash. It's main use cases is to cache |
|
10 | its content and not affecting its hash. It's main use cases is to cache | |
11 | important information related to a changesets. |
|
11 | important information related to a changesets. | |
12 |
|
12 | |||
13 | The current implementation is experimental and subject to changes. Do not rely |
|
13 | The current implementation is experimental and subject to changes. Do not rely | |
14 | on it in production. |
|
14 | on it in production. | |
15 |
|
15 | |||
16 | Sidedata are stored in the revlog itself, withing the revision rawtext. They |
|
16 | Sidedata are stored in the revlog itself, withing the revision rawtext. They | |
17 | are inserted, removed from it using the flagprocessors mechanism. The following |
|
17 | are inserted, removed from it using the flagprocessors mechanism. The following | |
18 | format is currently used:: |
|
18 | format is currently used:: | |
19 |
|
19 | |||
20 | initial header: |
|
20 | initial header: | |
21 | <number of sidedata; 2 bytes> |
|
21 | <number of sidedata; 2 bytes> | |
22 | sidedata (repeated N times): |
|
22 | sidedata (repeated N times): | |
23 | <sidedata-key; 2 bytes> |
|
23 | <sidedata-key; 2 bytes> | |
24 | <sidedata-entry-length: 4 bytes> |
|
24 | <sidedata-entry-length: 4 bytes> | |
25 | <sidedata-content-sha1-digest: 20 bytes> |
|
25 | <sidedata-content-sha1-digest: 20 bytes> | |
26 | <sidedata-content; X bytes> |
|
26 | <sidedata-content; X bytes> | |
27 | normal raw text: |
|
27 | normal raw text: | |
28 | <all bytes remaining in the rawtext> |
|
28 | <all bytes remaining in the rawtext> | |
29 |
|
29 | |||
30 | This is a simple and effective format. It should be enought to experiment with |
|
30 | This is a simple and effective format. It should be enought to experiment with | |
31 | the concept. |
|
31 | the concept. | |
32 | """ |
|
32 | """ | |
33 |
|
33 | |||
34 | from __future__ import absolute_import |
|
34 | from __future__ import absolute_import | |
35 |
|
35 | |||
36 | import hashlib |
|
36 | import hashlib | |
37 | import struct |
|
37 | import struct | |
38 |
|
38 | |||
39 | from .. import error |
|
39 | from .. import error | |
40 |
|
40 | |||
|
41 | ## sidedata type constant | |||
|
42 | # reserve a block for testing purposes. | |||
|
43 | SD_TEST1 = 1 | |||
|
44 | SD_TEST2 = 2 | |||
|
45 | SD_TEST3 = 3 | |||
|
46 | SD_TEST4 = 4 | |||
|
47 | SD_TEST5 = 5 | |||
|
48 | SD_TEST6 = 6 | |||
|
49 | SD_TEST7 = 7 | |||
|
50 | ||||
|
51 | # internal format constant | |||
41 | SIDEDATA_HEADER = struct.Struct('>H') |
|
52 | SIDEDATA_HEADER = struct.Struct('>H') | |
42 | SIDEDATA_ENTRY = struct.Struct('>HL20s') |
|
53 | SIDEDATA_ENTRY = struct.Struct('>HL20s') | |
43 |
|
54 | |||
44 | def sidedatawriteprocessor(rl, text, sidedata): |
|
55 | def sidedatawriteprocessor(rl, text, sidedata): | |
45 | sidedata = list(sidedata.items()) |
|
56 | sidedata = list(sidedata.items()) | |
46 | sidedata.sort() |
|
57 | sidedata.sort() | |
47 | rawtext = [SIDEDATA_HEADER.pack(len(sidedata))] |
|
58 | rawtext = [SIDEDATA_HEADER.pack(len(sidedata))] | |
48 | for key, value in sidedata: |
|
59 | for key, value in sidedata: | |
49 | digest = hashlib.sha1(value).digest() |
|
60 | digest = hashlib.sha1(value).digest() | |
50 | rawtext.append(SIDEDATA_ENTRY.pack(key, len(value), digest)) |
|
61 | rawtext.append(SIDEDATA_ENTRY.pack(key, len(value), digest)) | |
51 | for key, value in sidedata: |
|
62 | for key, value in sidedata: | |
52 | rawtext.append(value) |
|
63 | rawtext.append(value) | |
53 | rawtext.append(bytes(text)) |
|
64 | rawtext.append(bytes(text)) | |
54 | return ''.join(rawtext), False |
|
65 | return ''.join(rawtext), False | |
55 |
|
66 | |||
56 | def sidedatareadprocessor(rl, text): |
|
67 | def sidedatareadprocessor(rl, text): | |
57 | sidedata = {} |
|
68 | sidedata = {} | |
58 | offset = 0 |
|
69 | offset = 0 | |
59 | nbentry, = SIDEDATA_HEADER.unpack(text[:SIDEDATA_HEADER.size]) |
|
70 | nbentry, = SIDEDATA_HEADER.unpack(text[:SIDEDATA_HEADER.size]) | |
60 | offset += SIDEDATA_HEADER.size |
|
71 | offset += SIDEDATA_HEADER.size | |
61 | dataoffset = SIDEDATA_HEADER.size + (SIDEDATA_ENTRY.size * nbentry) |
|
72 | dataoffset = SIDEDATA_HEADER.size + (SIDEDATA_ENTRY.size * nbentry) | |
62 | for i in range(nbentry): |
|
73 | for i in range(nbentry): | |
63 | nextoffset = offset + SIDEDATA_ENTRY.size |
|
74 | nextoffset = offset + SIDEDATA_ENTRY.size | |
64 | key, size, storeddigest = SIDEDATA_ENTRY.unpack(text[offset:nextoffset]) |
|
75 | key, size, storeddigest = SIDEDATA_ENTRY.unpack(text[offset:nextoffset]) | |
65 | offset = nextoffset |
|
76 | offset = nextoffset | |
66 | # read the data associated with that entry |
|
77 | # read the data associated with that entry | |
67 | nextdataoffset = dataoffset + size |
|
78 | nextdataoffset = dataoffset + size | |
68 | entrytext = text[dataoffset:nextdataoffset] |
|
79 | entrytext = text[dataoffset:nextdataoffset] | |
69 | readdigest = hashlib.sha1(entrytext).digest() |
|
80 | readdigest = hashlib.sha1(entrytext).digest() | |
70 | if storeddigest != readdigest: |
|
81 | if storeddigest != readdigest: | |
71 | raise error.SidedataHashError(key, storeddigest, readdigest) |
|
82 | raise error.SidedataHashError(key, storeddigest, readdigest) | |
72 | sidedata[key] = entrytext |
|
83 | sidedata[key] = entrytext | |
73 | dataoffset = nextdataoffset |
|
84 | dataoffset = nextdataoffset | |
74 | text = text[dataoffset:] |
|
85 | text = text[dataoffset:] | |
75 | return text, True, sidedata |
|
86 | return text, True, sidedata | |
76 |
|
87 | |||
77 | def sidedatarawprocessor(rl, text): |
|
88 | def sidedatarawprocessor(rl, text): | |
78 | # side data modifies rawtext and prevent rawtext hash validation |
|
89 | # side data modifies rawtext and prevent rawtext hash validation | |
79 | return False |
|
90 | return False | |
80 |
|
91 | |||
81 | processors = ( |
|
92 | processors = ( | |
82 | sidedatareadprocessor, |
|
93 | sidedatareadprocessor, | |
83 | sidedatawriteprocessor, |
|
94 | sidedatawriteprocessor, | |
84 | sidedatarawprocessor, |
|
95 | sidedatarawprocessor, | |
85 | ) |
|
96 | ) |
@@ -1,65 +1,83 b'' | |||||
1 | ========================================================== |
|
1 | ========================================================== | |
2 | Test file dedicated to checking side-data related behavior |
|
2 | Test file dedicated to checking side-data related behavior | |
3 | ========================================================== |
|
3 | ========================================================== | |
4 |
|
4 | |||
|
5 | Check data can be written/read from sidedata | |||
|
6 | ============================================ | |||
|
7 | ||||
|
8 | $ cat << EOF >> $HGRCPATH | |||
|
9 | > [extensions] | |||
|
10 | > testsidedata=$TESTDIR/testlib/ext-sidedata.py | |||
|
11 | > EOF | |||
|
12 | ||||
|
13 | $ hg init test-sidedata --config format.use-side-data=yes | |||
|
14 | $ cd test-sidedata | |||
|
15 | $ echo aaa > a | |||
|
16 | $ hg add a | |||
|
17 | $ hg commit -m a --traceback | |||
|
18 | $ echo aaa > b | |||
|
19 | $ hg add b | |||
|
20 | $ hg commit -m b | |||
|
21 | $ echo xxx >> a | |||
|
22 | $ hg commit -m aa | |||
5 |
|
23 | |||
6 | Check upgrade behavior |
|
24 | Check upgrade behavior | |
7 | ====================== |
|
25 | ====================== | |
8 |
|
26 | |||
9 | Right now, sidedata has not upgrade support |
|
27 | Right now, sidedata has not upgrade support | |
10 |
|
28 | |||
11 | Check that we cannot upgrade to sidedata |
|
29 | Check that we cannot upgrade to sidedata | |
12 | ---------------------------------------- |
|
30 | ---------------------------------------- | |
13 |
|
31 | |||
14 | $ hg init up-no-side-data --config format.use-side-data=no |
|
32 | $ hg init up-no-side-data --config format.use-side-data=no | |
15 | $ hg debugformat -v -R up-no-side-data |
|
33 | $ hg debugformat -v -R up-no-side-data | |
16 | format-variant repo config default |
|
34 | format-variant repo config default | |
17 | fncache: yes yes yes |
|
35 | fncache: yes yes yes | |
18 | dotencode: yes yes yes |
|
36 | dotencode: yes yes yes | |
19 | generaldelta: yes yes yes |
|
37 | generaldelta: yes yes yes | |
20 | sparserevlog: yes yes yes |
|
38 | sparserevlog: yes yes yes | |
21 | sidedata: no no no |
|
39 | sidedata: no no no | |
22 | plain-cl-delta: yes yes yes |
|
40 | plain-cl-delta: yes yes yes | |
23 | compression: zlib zlib zlib |
|
41 | compression: zlib zlib zlib | |
24 | compression-level: default default default |
|
42 | compression-level: default default default | |
25 | $ hg debugformat -v -R up-no-side-data --config format.use-side-data=yes |
|
43 | $ hg debugformat -v -R up-no-side-data --config format.use-side-data=yes | |
26 | format-variant repo config default |
|
44 | format-variant repo config default | |
27 | fncache: yes yes yes |
|
45 | fncache: yes yes yes | |
28 | dotencode: yes yes yes |
|
46 | dotencode: yes yes yes | |
29 | generaldelta: yes yes yes |
|
47 | generaldelta: yes yes yes | |
30 | sparserevlog: yes yes yes |
|
48 | sparserevlog: yes yes yes | |
31 | sidedata: no yes no |
|
49 | sidedata: no yes no | |
32 | plain-cl-delta: yes yes yes |
|
50 | plain-cl-delta: yes yes yes | |
33 | compression: zlib zlib zlib |
|
51 | compression: zlib zlib zlib | |
34 | compression-level: default default default |
|
52 | compression-level: default default default | |
35 | $ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes |
|
53 | $ hg debugupgraderepo -R up-no-side-data --config format.use-side-data=yes | |
36 | abort: cannot upgrade repository; do not support adding requirement: exp-sidedata-flag |
|
54 | abort: cannot upgrade repository; do not support adding requirement: exp-sidedata-flag | |
37 | [255] |
|
55 | [255] | |
38 |
|
56 | |||
39 | Check that we cannot upgrade to sidedata |
|
57 | Check that we cannot upgrade to sidedata | |
40 | ---------------------------------------- |
|
58 | ---------------------------------------- | |
41 |
|
59 | |||
42 | $ hg init up-side-data --config format.use-side-data=yes |
|
60 | $ hg init up-side-data --config format.use-side-data=yes | |
43 | $ hg debugformat -v -R up-side-data |
|
61 | $ hg debugformat -v -R up-side-data | |
44 | format-variant repo config default |
|
62 | format-variant repo config default | |
45 | fncache: yes yes yes |
|
63 | fncache: yes yes yes | |
46 | dotencode: yes yes yes |
|
64 | dotencode: yes yes yes | |
47 | generaldelta: yes yes yes |
|
65 | generaldelta: yes yes yes | |
48 | sparserevlog: yes yes yes |
|
66 | sparserevlog: yes yes yes | |
49 | sidedata: yes no no |
|
67 | sidedata: yes no no | |
50 | plain-cl-delta: yes yes yes |
|
68 | plain-cl-delta: yes yes yes | |
51 | compression: zlib zlib zlib |
|
69 | compression: zlib zlib zlib | |
52 | compression-level: default default default |
|
70 | compression-level: default default default | |
53 | $ hg debugformat -v -R up-side-data --config format.use-side-data=no |
|
71 | $ hg debugformat -v -R up-side-data --config format.use-side-data=no | |
54 | format-variant repo config default |
|
72 | format-variant repo config default | |
55 | fncache: yes yes yes |
|
73 | fncache: yes yes yes | |
56 | dotencode: yes yes yes |
|
74 | dotencode: yes yes yes | |
57 | generaldelta: yes yes yes |
|
75 | generaldelta: yes yes yes | |
58 | sparserevlog: yes yes yes |
|
76 | sparserevlog: yes yes yes | |
59 | sidedata: yes no no |
|
77 | sidedata: yes no no | |
60 | plain-cl-delta: yes yes yes |
|
78 | plain-cl-delta: yes yes yes | |
61 | compression: zlib zlib zlib |
|
79 | compression: zlib zlib zlib | |
62 | compression-level: default default default |
|
80 | compression-level: default default default | |
63 | $ hg debugupgraderepo -R up-side-data --config format.use-side-data=no |
|
81 | $ hg debugupgraderepo -R up-side-data --config format.use-side-data=no | |
64 | abort: cannot upgrade repository; requirement would be removed: exp-sidedata-flag |
|
82 | abort: cannot upgrade repository; requirement would be removed: exp-sidedata-flag | |
65 | [255] |
|
83 | [255] |
General Comments 0
You need to be logged in to leave comments.
Login now