Show More
@@ -1,549 +1,556 b'' | |||||
1 | # storageutil.py - Storage functionality agnostic of backend implementation. |
|
1 | # storageutil.py - Storage functionality agnostic of backend implementation. | |
2 | # |
|
2 | # | |
3 | # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> |
|
3 | # Copyright 2018 Gregory Szorc <gregory.szorc@gmail.com> | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 |
|
7 | |||
8 | from __future__ import absolute_import |
|
8 | from __future__ import absolute_import | |
9 |
|
9 | |||
10 | import re |
|
10 | import re | |
11 | import struct |
|
11 | import struct | |
12 |
|
12 | |||
13 | from ..i18n import _ |
|
13 | from ..i18n import _ | |
14 | from ..node import ( |
|
14 | from ..node import ( | |
15 | bin, |
|
15 | bin, | |
16 | nullrev, |
|
16 | nullrev, | |
17 | sha1nodeconstants, |
|
17 | sha1nodeconstants, | |
18 | ) |
|
18 | ) | |
19 | from .. import ( |
|
19 | from .. import ( | |
20 | dagop, |
|
20 | dagop, | |
21 | error, |
|
21 | error, | |
22 | mdiff, |
|
22 | mdiff, | |
23 | pycompat, |
|
23 | pycompat, | |
24 | ) |
|
24 | ) | |
25 | from ..interfaces import repository |
|
25 | from ..interfaces import repository | |
26 | from ..revlogutils import sidedata as sidedatamod |
|
26 | from ..revlogutils import sidedata as sidedatamod | |
27 | from ..utils import hashutil |
|
27 | from ..utils import hashutil | |
28 |
|
28 | |||
29 | _nullhash = hashutil.sha1(sha1nodeconstants.nullid) |
|
29 | _nullhash = hashutil.sha1(sha1nodeconstants.nullid) | |
30 |
|
30 | |||
31 | # revision data contains extra metadata not part of the official digest |
|
31 | # revision data contains extra metadata not part of the official digest | |
32 | # Only used in changegroup >= v4. |
|
32 | # Only used in changegroup >= v4. | |
33 | CG_FLAG_SIDEDATA = 1 |
|
33 | CG_FLAG_SIDEDATA = 1 | |
34 |
|
34 | |||
35 |
|
35 | |||
36 | def hashrevisionsha1(text, p1, p2): |
|
36 | def hashrevisionsha1(text, p1, p2): | |
37 | """Compute the SHA-1 for revision data and its parents. |
|
37 | """Compute the SHA-1 for revision data and its parents. | |
38 |
|
38 | |||
39 | This hash combines both the current file contents and its history |
|
39 | This hash combines both the current file contents and its history | |
40 | in a manner that makes it easy to distinguish nodes with the same |
|
40 | in a manner that makes it easy to distinguish nodes with the same | |
41 | content in the revision graph. |
|
41 | content in the revision graph. | |
42 | """ |
|
42 | """ | |
43 | # As of now, if one of the parent node is null, p2 is null |
|
43 | # As of now, if one of the parent node is null, p2 is null | |
44 | if p2 == sha1nodeconstants.nullid: |
|
44 | if p2 == sha1nodeconstants.nullid: | |
45 | # deep copy of a hash is faster than creating one |
|
45 | # deep copy of a hash is faster than creating one | |
46 | s = _nullhash.copy() |
|
46 | s = _nullhash.copy() | |
47 | s.update(p1) |
|
47 | s.update(p1) | |
48 | else: |
|
48 | else: | |
49 | # none of the parent nodes are nullid |
|
49 | # none of the parent nodes are nullid | |
50 | if p1 < p2: |
|
50 | if p1 < p2: | |
51 | a = p1 |
|
51 | a = p1 | |
52 | b = p2 |
|
52 | b = p2 | |
53 | else: |
|
53 | else: | |
54 | a = p2 |
|
54 | a = p2 | |
55 | b = p1 |
|
55 | b = p1 | |
56 | s = hashutil.sha1(a) |
|
56 | s = hashutil.sha1(a) | |
57 | s.update(b) |
|
57 | s.update(b) | |
58 | s.update(text) |
|
58 | s.update(text) | |
59 | return s.digest() |
|
59 | return s.digest() | |
60 |
|
60 | |||
61 |
|
61 | |||
62 | METADATA_RE = re.compile(b'\x01\n') |
|
62 | METADATA_RE = re.compile(b'\x01\n') | |
63 |
|
63 | |||
64 |
|
64 | |||
65 | def parsemeta(text): |
|
65 | def parsemeta(text): | |
66 | """Parse metadata header from revision data. |
|
66 | """Parse metadata header from revision data. | |
67 |
|
67 | |||
68 | Returns a 2-tuple of (metadata, offset), where both can be None if there |
|
68 | Returns a 2-tuple of (metadata, offset), where both can be None if there | |
69 | is no metadata. |
|
69 | is no metadata. | |
70 | """ |
|
70 | """ | |
71 | # text can be buffer, so we can't use .startswith or .index |
|
71 | # text can be buffer, so we can't use .startswith or .index | |
72 | if text[:2] != b'\x01\n': |
|
72 | if text[:2] != b'\x01\n': | |
73 | return None, None |
|
73 | return None, None | |
74 | s = METADATA_RE.search(text, 2).start() |
|
74 | s = METADATA_RE.search(text, 2).start() | |
75 | mtext = text[2:s] |
|
75 | mtext = text[2:s] | |
76 | meta = {} |
|
76 | meta = {} | |
77 | for l in mtext.splitlines(): |
|
77 | for l in mtext.splitlines(): | |
78 | k, v = l.split(b': ', 1) |
|
78 | k, v = l.split(b': ', 1) | |
79 | meta[k] = v |
|
79 | meta[k] = v | |
80 | return meta, s + 2 |
|
80 | return meta, s + 2 | |
81 |
|
81 | |||
82 |
|
82 | |||
83 | def packmeta(meta, text): |
|
83 | def packmeta(meta, text): | |
84 | """Add metadata to fulltext to produce revision text.""" |
|
84 | """Add metadata to fulltext to produce revision text.""" | |
85 | keys = sorted(meta) |
|
85 | keys = sorted(meta) | |
86 | metatext = b''.join(b'%s: %s\n' % (k, meta[k]) for k in keys) |
|
86 | metatext = b''.join(b'%s: %s\n' % (k, meta[k]) for k in keys) | |
87 | return b'\x01\n%s\x01\n%s' % (metatext, text) |
|
87 | return b'\x01\n%s\x01\n%s' % (metatext, text) | |
88 |
|
88 | |||
89 |
|
89 | |||
90 | def iscensoredtext(text): |
|
90 | def iscensoredtext(text): | |
91 | meta = parsemeta(text)[0] |
|
91 | meta = parsemeta(text)[0] | |
92 | return meta and b'censored' in meta |
|
92 | return meta and b'censored' in meta | |
93 |
|
93 | |||
94 |
|
94 | |||
95 | def filtermetadata(text): |
|
95 | def filtermetadata(text): | |
96 | """Extract just the revision data from source text. |
|
96 | """Extract just the revision data from source text. | |
97 |
|
97 | |||
98 | Returns ``text`` unless it has a metadata header, in which case we return |
|
98 | Returns ``text`` unless it has a metadata header, in which case we return | |
99 | a new buffer without hte metadata. |
|
99 | a new buffer without hte metadata. | |
100 | """ |
|
100 | """ | |
101 | if not text.startswith(b'\x01\n'): |
|
101 | if not text.startswith(b'\x01\n'): | |
102 | return text |
|
102 | return text | |
103 |
|
103 | |||
104 | offset = text.index(b'\x01\n', 2) |
|
104 | offset = text.index(b'\x01\n', 2) | |
105 | return text[offset + 2 :] |
|
105 | return text[offset + 2 :] | |
106 |
|
106 | |||
107 |
|
107 | |||
108 | def filerevisioncopied(store, node): |
|
108 | def filerevisioncopied(store, node): | |
109 | """Resolve file revision copy metadata. |
|
109 | """Resolve file revision copy metadata. | |
110 |
|
110 | |||
111 | Returns ``False`` if the file has no copy metadata. Otherwise a |
|
111 | Returns ``False`` if the file has no copy metadata. Otherwise a | |
112 | 2-tuple of the source filename and node. |
|
112 | 2-tuple of the source filename and node. | |
113 | """ |
|
113 | """ | |
114 | if store.parents(node)[0] != sha1nodeconstants.nullid: |
|
114 | if store.parents(node)[0] != sha1nodeconstants.nullid: | |
|
115 | # When creating a copy or move we set filelog parents to null, | |||
|
116 | # because contents are probably unrelated and making a delta | |||
|
117 | # would not be useful. | |||
|
118 | # Conversely, if filelog p1 is non-null we know | |||
|
119 | # there is no copy metadata. | |||
|
120 | # In the presence of merges, this reasoning becomes invalid | |||
|
121 | # if we reorder parents. See tests/test-issue6528.t. | |||
115 | return False |
|
122 | return False | |
116 |
|
123 | |||
117 | meta = parsemeta(store.revision(node))[0] |
|
124 | meta = parsemeta(store.revision(node))[0] | |
118 |
|
125 | |||
119 | # copy and copyrev occur in pairs. In rare cases due to old bugs, |
|
126 | # copy and copyrev occur in pairs. In rare cases due to old bugs, | |
120 | # one can occur without the other. So ensure both are present to flag |
|
127 | # one can occur without the other. So ensure both are present to flag | |
121 | # as a copy. |
|
128 | # as a copy. | |
122 | if meta and b'copy' in meta and b'copyrev' in meta: |
|
129 | if meta and b'copy' in meta and b'copyrev' in meta: | |
123 | return meta[b'copy'], bin(meta[b'copyrev']) |
|
130 | return meta[b'copy'], bin(meta[b'copyrev']) | |
124 |
|
131 | |||
125 | return False |
|
132 | return False | |
126 |
|
133 | |||
127 |
|
134 | |||
128 | def filedataequivalent(store, node, filedata): |
|
135 | def filedataequivalent(store, node, filedata): | |
129 | """Determines whether file data is equivalent to a stored node. |
|
136 | """Determines whether file data is equivalent to a stored node. | |
130 |
|
137 | |||
131 | Returns True if the passed file data would hash to the same value |
|
138 | Returns True if the passed file data would hash to the same value | |
132 | as a stored revision and False otherwise. |
|
139 | as a stored revision and False otherwise. | |
133 |
|
140 | |||
134 | When a stored revision is censored, filedata must be empty to have |
|
141 | When a stored revision is censored, filedata must be empty to have | |
135 | equivalence. |
|
142 | equivalence. | |
136 |
|
143 | |||
137 | When a stored revision has copy metadata, it is ignored as part |
|
144 | When a stored revision has copy metadata, it is ignored as part | |
138 | of the compare. |
|
145 | of the compare. | |
139 | """ |
|
146 | """ | |
140 |
|
147 | |||
141 | if filedata.startswith(b'\x01\n'): |
|
148 | if filedata.startswith(b'\x01\n'): | |
142 | revisiontext = b'\x01\n\x01\n' + filedata |
|
149 | revisiontext = b'\x01\n\x01\n' + filedata | |
143 | else: |
|
150 | else: | |
144 | revisiontext = filedata |
|
151 | revisiontext = filedata | |
145 |
|
152 | |||
146 | p1, p2 = store.parents(node) |
|
153 | p1, p2 = store.parents(node) | |
147 |
|
154 | |||
148 | computednode = hashrevisionsha1(revisiontext, p1, p2) |
|
155 | computednode = hashrevisionsha1(revisiontext, p1, p2) | |
149 |
|
156 | |||
150 | if computednode == node: |
|
157 | if computednode == node: | |
151 | return True |
|
158 | return True | |
152 |
|
159 | |||
153 | # Censored files compare against the empty file. |
|
160 | # Censored files compare against the empty file. | |
154 | if store.iscensored(store.rev(node)): |
|
161 | if store.iscensored(store.rev(node)): | |
155 | return filedata == b'' |
|
162 | return filedata == b'' | |
156 |
|
163 | |||
157 | # Renaming a file produces a different hash, even if the data |
|
164 | # Renaming a file produces a different hash, even if the data | |
158 | # remains unchanged. Check if that's the case. |
|
165 | # remains unchanged. Check if that's the case. | |
159 | if store.renamed(node): |
|
166 | if store.renamed(node): | |
160 | return store.read(node) == filedata |
|
167 | return store.read(node) == filedata | |
161 |
|
168 | |||
162 | return False |
|
169 | return False | |
163 |
|
170 | |||
164 |
|
171 | |||
165 | def iterrevs(storelen, start=0, stop=None): |
|
172 | def iterrevs(storelen, start=0, stop=None): | |
166 | """Iterate over revision numbers in a store.""" |
|
173 | """Iterate over revision numbers in a store.""" | |
167 | step = 1 |
|
174 | step = 1 | |
168 |
|
175 | |||
169 | if stop is not None: |
|
176 | if stop is not None: | |
170 | if start > stop: |
|
177 | if start > stop: | |
171 | step = -1 |
|
178 | step = -1 | |
172 | stop += step |
|
179 | stop += step | |
173 | if stop > storelen: |
|
180 | if stop > storelen: | |
174 | stop = storelen |
|
181 | stop = storelen | |
175 | else: |
|
182 | else: | |
176 | stop = storelen |
|
183 | stop = storelen | |
177 |
|
184 | |||
178 | return pycompat.xrange(start, stop, step) |
|
185 | return pycompat.xrange(start, stop, step) | |
179 |
|
186 | |||
180 |
|
187 | |||
181 | def fileidlookup(store, fileid, identifier): |
|
188 | def fileidlookup(store, fileid, identifier): | |
182 | """Resolve the file node for a value. |
|
189 | """Resolve the file node for a value. | |
183 |
|
190 | |||
184 | ``store`` is an object implementing the ``ifileindex`` interface. |
|
191 | ``store`` is an object implementing the ``ifileindex`` interface. | |
185 |
|
192 | |||
186 | ``fileid`` can be: |
|
193 | ``fileid`` can be: | |
187 |
|
194 | |||
188 | * A 20 or 32 byte binary node. |
|
195 | * A 20 or 32 byte binary node. | |
189 | * An integer revision number |
|
196 | * An integer revision number | |
190 | * A 40 or 64 byte hex node. |
|
197 | * A 40 or 64 byte hex node. | |
191 | * A bytes that can be parsed as an integer representing a revision number. |
|
198 | * A bytes that can be parsed as an integer representing a revision number. | |
192 |
|
199 | |||
193 | ``identifier`` is used to populate ``error.LookupError`` with an identifier |
|
200 | ``identifier`` is used to populate ``error.LookupError`` with an identifier | |
194 | for the store. |
|
201 | for the store. | |
195 |
|
202 | |||
196 | Raises ``error.LookupError`` on failure. |
|
203 | Raises ``error.LookupError`` on failure. | |
197 | """ |
|
204 | """ | |
198 | if isinstance(fileid, int): |
|
205 | if isinstance(fileid, int): | |
199 | try: |
|
206 | try: | |
200 | return store.node(fileid) |
|
207 | return store.node(fileid) | |
201 | except IndexError: |
|
208 | except IndexError: | |
202 | raise error.LookupError( |
|
209 | raise error.LookupError( | |
203 | b'%d' % fileid, identifier, _(b'no match found') |
|
210 | b'%d' % fileid, identifier, _(b'no match found') | |
204 | ) |
|
211 | ) | |
205 |
|
212 | |||
206 | if len(fileid) in (20, 32): |
|
213 | if len(fileid) in (20, 32): | |
207 | try: |
|
214 | try: | |
208 | store.rev(fileid) |
|
215 | store.rev(fileid) | |
209 | return fileid |
|
216 | return fileid | |
210 | except error.LookupError: |
|
217 | except error.LookupError: | |
211 | pass |
|
218 | pass | |
212 |
|
219 | |||
213 | if len(fileid) in (40, 64): |
|
220 | if len(fileid) in (40, 64): | |
214 | try: |
|
221 | try: | |
215 | rawnode = bin(fileid) |
|
222 | rawnode = bin(fileid) | |
216 | store.rev(rawnode) |
|
223 | store.rev(rawnode) | |
217 | return rawnode |
|
224 | return rawnode | |
218 | except TypeError: |
|
225 | except TypeError: | |
219 | pass |
|
226 | pass | |
220 |
|
227 | |||
221 | try: |
|
228 | try: | |
222 | rev = int(fileid) |
|
229 | rev = int(fileid) | |
223 |
|
230 | |||
224 | if b'%d' % rev != fileid: |
|
231 | if b'%d' % rev != fileid: | |
225 | raise ValueError |
|
232 | raise ValueError | |
226 |
|
233 | |||
227 | try: |
|
234 | try: | |
228 | return store.node(rev) |
|
235 | return store.node(rev) | |
229 | except (IndexError, TypeError): |
|
236 | except (IndexError, TypeError): | |
230 | pass |
|
237 | pass | |
231 | except (ValueError, OverflowError): |
|
238 | except (ValueError, OverflowError): | |
232 | pass |
|
239 | pass | |
233 |
|
240 | |||
234 | raise error.LookupError(fileid, identifier, _(b'no match found')) |
|
241 | raise error.LookupError(fileid, identifier, _(b'no match found')) | |
235 |
|
242 | |||
236 |
|
243 | |||
237 | def resolvestripinfo(minlinkrev, tiprev, headrevs, linkrevfn, parentrevsfn): |
|
244 | def resolvestripinfo(minlinkrev, tiprev, headrevs, linkrevfn, parentrevsfn): | |
238 | """Resolve information needed to strip revisions. |
|
245 | """Resolve information needed to strip revisions. | |
239 |
|
246 | |||
240 | Finds the minimum revision number that must be stripped in order to |
|
247 | Finds the minimum revision number that must be stripped in order to | |
241 | strip ``minlinkrev``. |
|
248 | strip ``minlinkrev``. | |
242 |
|
249 | |||
243 | Returns a 2-tuple of the minimum revision number to do that and a set |
|
250 | Returns a 2-tuple of the minimum revision number to do that and a set | |
244 | of all revision numbers that have linkrevs that would be broken |
|
251 | of all revision numbers that have linkrevs that would be broken | |
245 | by that strip. |
|
252 | by that strip. | |
246 |
|
253 | |||
247 | ``tiprev`` is the current tip-most revision. It is ``len(store) - 1``. |
|
254 | ``tiprev`` is the current tip-most revision. It is ``len(store) - 1``. | |
248 | ``headrevs`` is an iterable of head revisions. |
|
255 | ``headrevs`` is an iterable of head revisions. | |
249 | ``linkrevfn`` is a callable that receives a revision and returns a linked |
|
256 | ``linkrevfn`` is a callable that receives a revision and returns a linked | |
250 | revision. |
|
257 | revision. | |
251 | ``parentrevsfn`` is a callable that receives a revision number and returns |
|
258 | ``parentrevsfn`` is a callable that receives a revision number and returns | |
252 | an iterable of its parent revision numbers. |
|
259 | an iterable of its parent revision numbers. | |
253 | """ |
|
260 | """ | |
254 | brokenrevs = set() |
|
261 | brokenrevs = set() | |
255 | strippoint = tiprev + 1 |
|
262 | strippoint = tiprev + 1 | |
256 |
|
263 | |||
257 | heads = {} |
|
264 | heads = {} | |
258 | futurelargelinkrevs = set() |
|
265 | futurelargelinkrevs = set() | |
259 | for head in headrevs: |
|
266 | for head in headrevs: | |
260 | headlinkrev = linkrevfn(head) |
|
267 | headlinkrev = linkrevfn(head) | |
261 | heads[head] = headlinkrev |
|
268 | heads[head] = headlinkrev | |
262 | if headlinkrev >= minlinkrev: |
|
269 | if headlinkrev >= minlinkrev: | |
263 | futurelargelinkrevs.add(headlinkrev) |
|
270 | futurelargelinkrevs.add(headlinkrev) | |
264 |
|
271 | |||
265 | # This algorithm involves walking down the rev graph, starting at the |
|
272 | # This algorithm involves walking down the rev graph, starting at the | |
266 | # heads. Since the revs are topologically sorted according to linkrev, |
|
273 | # heads. Since the revs are topologically sorted according to linkrev, | |
267 | # once all head linkrevs are below the minlink, we know there are |
|
274 | # once all head linkrevs are below the minlink, we know there are | |
268 | # no more revs that could have a linkrev greater than minlink. |
|
275 | # no more revs that could have a linkrev greater than minlink. | |
269 | # So we can stop walking. |
|
276 | # So we can stop walking. | |
270 | while futurelargelinkrevs: |
|
277 | while futurelargelinkrevs: | |
271 | strippoint -= 1 |
|
278 | strippoint -= 1 | |
272 | linkrev = heads.pop(strippoint) |
|
279 | linkrev = heads.pop(strippoint) | |
273 |
|
280 | |||
274 | if linkrev < minlinkrev: |
|
281 | if linkrev < minlinkrev: | |
275 | brokenrevs.add(strippoint) |
|
282 | brokenrevs.add(strippoint) | |
276 | else: |
|
283 | else: | |
277 | futurelargelinkrevs.remove(linkrev) |
|
284 | futurelargelinkrevs.remove(linkrev) | |
278 |
|
285 | |||
279 | for p in parentrevsfn(strippoint): |
|
286 | for p in parentrevsfn(strippoint): | |
280 | if p != nullrev: |
|
287 | if p != nullrev: | |
281 | plinkrev = linkrevfn(p) |
|
288 | plinkrev = linkrevfn(p) | |
282 | heads[p] = plinkrev |
|
289 | heads[p] = plinkrev | |
283 | if plinkrev >= minlinkrev: |
|
290 | if plinkrev >= minlinkrev: | |
284 | futurelargelinkrevs.add(plinkrev) |
|
291 | futurelargelinkrevs.add(plinkrev) | |
285 |
|
292 | |||
286 | return strippoint, brokenrevs |
|
293 | return strippoint, brokenrevs | |
287 |
|
294 | |||
288 |
|
295 | |||
289 | def emitrevisions( |
|
296 | def emitrevisions( | |
290 | store, |
|
297 | store, | |
291 | nodes, |
|
298 | nodes, | |
292 | nodesorder, |
|
299 | nodesorder, | |
293 | resultcls, |
|
300 | resultcls, | |
294 | deltaparentfn=None, |
|
301 | deltaparentfn=None, | |
295 | candeltafn=None, |
|
302 | candeltafn=None, | |
296 | rawsizefn=None, |
|
303 | rawsizefn=None, | |
297 | revdifffn=None, |
|
304 | revdifffn=None, | |
298 | flagsfn=None, |
|
305 | flagsfn=None, | |
299 | deltamode=repository.CG_DELTAMODE_STD, |
|
306 | deltamode=repository.CG_DELTAMODE_STD, | |
300 | revisiondata=False, |
|
307 | revisiondata=False, | |
301 | assumehaveparentrevisions=False, |
|
308 | assumehaveparentrevisions=False, | |
302 | sidedata_helpers=None, |
|
309 | sidedata_helpers=None, | |
303 | ): |
|
310 | ): | |
304 | """Generic implementation of ifiledata.emitrevisions(). |
|
311 | """Generic implementation of ifiledata.emitrevisions(). | |
305 |
|
312 | |||
306 | Emitting revision data is subtly complex. This function attempts to |
|
313 | Emitting revision data is subtly complex. This function attempts to | |
307 | encapsulate all the logic for doing so in a backend-agnostic way. |
|
314 | encapsulate all the logic for doing so in a backend-agnostic way. | |
308 |
|
315 | |||
309 | ``store`` |
|
316 | ``store`` | |
310 | Object conforming to ``ifilestorage`` interface. |
|
317 | Object conforming to ``ifilestorage`` interface. | |
311 |
|
318 | |||
312 | ``nodes`` |
|
319 | ``nodes`` | |
313 | List of revision nodes whose data to emit. |
|
320 | List of revision nodes whose data to emit. | |
314 |
|
321 | |||
315 | ``resultcls`` |
|
322 | ``resultcls`` | |
316 | A type implementing the ``irevisiondelta`` interface that will be |
|
323 | A type implementing the ``irevisiondelta`` interface that will be | |
317 | constructed and returned. |
|
324 | constructed and returned. | |
318 |
|
325 | |||
319 | ``deltaparentfn`` (optional) |
|
326 | ``deltaparentfn`` (optional) | |
320 | Callable receiving a revision number and returning the revision number |
|
327 | Callable receiving a revision number and returning the revision number | |
321 | of a revision that the internal delta is stored against. This delta |
|
328 | of a revision that the internal delta is stored against. This delta | |
322 | will be preferred over computing a new arbitrary delta. |
|
329 | will be preferred over computing a new arbitrary delta. | |
323 |
|
330 | |||
324 | If not defined, a delta will always be computed from raw revision |
|
331 | If not defined, a delta will always be computed from raw revision | |
325 | data. |
|
332 | data. | |
326 |
|
333 | |||
327 | ``candeltafn`` (optional) |
|
334 | ``candeltafn`` (optional) | |
328 | Callable receiving a pair of revision numbers that returns a bool |
|
335 | Callable receiving a pair of revision numbers that returns a bool | |
329 | indicating whether a delta between them can be produced. |
|
336 | indicating whether a delta between them can be produced. | |
330 |
|
337 | |||
331 | If not defined, it is assumed that any two revisions can delta with |
|
338 | If not defined, it is assumed that any two revisions can delta with | |
332 | each other. |
|
339 | each other. | |
333 |
|
340 | |||
334 | ``rawsizefn`` (optional) |
|
341 | ``rawsizefn`` (optional) | |
335 | Callable receiving a revision number and returning the length of the |
|
342 | Callable receiving a revision number and returning the length of the | |
336 | ``store.rawdata(rev)``. |
|
343 | ``store.rawdata(rev)``. | |
337 |
|
344 | |||
338 | If not defined, ``len(store.rawdata(rev))`` will be called. |
|
345 | If not defined, ``len(store.rawdata(rev))`` will be called. | |
339 |
|
346 | |||
340 | ``revdifffn`` (optional) |
|
347 | ``revdifffn`` (optional) | |
341 | Callable receiving a pair of revision numbers that returns a delta |
|
348 | Callable receiving a pair of revision numbers that returns a delta | |
342 | between them. |
|
349 | between them. | |
343 |
|
350 | |||
344 | If not defined, a delta will be computed by invoking mdiff code |
|
351 | If not defined, a delta will be computed by invoking mdiff code | |
345 | on ``store.revision()`` results. |
|
352 | on ``store.revision()`` results. | |
346 |
|
353 | |||
347 | Defining this function allows a precomputed or stored delta to be |
|
354 | Defining this function allows a precomputed or stored delta to be | |
348 | used without having to compute on. |
|
355 | used without having to compute on. | |
349 |
|
356 | |||
350 | ``flagsfn`` (optional) |
|
357 | ``flagsfn`` (optional) | |
351 | Callable receiving a revision number and returns the integer flags |
|
358 | Callable receiving a revision number and returns the integer flags | |
352 | value for it. If not defined, flags value will be 0. |
|
359 | value for it. If not defined, flags value will be 0. | |
353 |
|
360 | |||
354 | ``deltamode`` |
|
361 | ``deltamode`` | |
355 | constaint on delta to be sent: |
|
362 | constaint on delta to be sent: | |
356 | * CG_DELTAMODE_STD - normal mode, try to reuse storage deltas, |
|
363 | * CG_DELTAMODE_STD - normal mode, try to reuse storage deltas, | |
357 | * CG_DELTAMODE_PREV - only delta against "prev", |
|
364 | * CG_DELTAMODE_PREV - only delta against "prev", | |
358 | * CG_DELTAMODE_FULL - only issue full snapshot. |
|
365 | * CG_DELTAMODE_FULL - only issue full snapshot. | |
359 |
|
366 | |||
360 | Whether to send fulltext revisions instead of deltas, if allowed. |
|
367 | Whether to send fulltext revisions instead of deltas, if allowed. | |
361 |
|
368 | |||
362 | ``nodesorder`` |
|
369 | ``nodesorder`` | |
363 | ``revisiondata`` |
|
370 | ``revisiondata`` | |
364 | ``assumehaveparentrevisions`` |
|
371 | ``assumehaveparentrevisions`` | |
365 | ``sidedata_helpers`` (optional) |
|
372 | ``sidedata_helpers`` (optional) | |
366 | If not None, means that sidedata should be included. |
|
373 | If not None, means that sidedata should be included. | |
367 | See `revlogutil.sidedata.get_sidedata_helpers`. |
|
374 | See `revlogutil.sidedata.get_sidedata_helpers`. | |
368 | """ |
|
375 | """ | |
369 |
|
376 | |||
370 | fnode = store.node |
|
377 | fnode = store.node | |
371 | frev = store.rev |
|
378 | frev = store.rev | |
372 |
|
379 | |||
373 | if nodesorder == b'nodes': |
|
380 | if nodesorder == b'nodes': | |
374 | revs = [frev(n) for n in nodes] |
|
381 | revs = [frev(n) for n in nodes] | |
375 | elif nodesorder == b'linear': |
|
382 | elif nodesorder == b'linear': | |
376 | revs = {frev(n) for n in nodes} |
|
383 | revs = {frev(n) for n in nodes} | |
377 | revs = dagop.linearize(revs, store.parentrevs) |
|
384 | revs = dagop.linearize(revs, store.parentrevs) | |
378 | else: # storage and default |
|
385 | else: # storage and default | |
379 | revs = sorted(frev(n) for n in nodes) |
|
386 | revs = sorted(frev(n) for n in nodes) | |
380 |
|
387 | |||
381 | prevrev = None |
|
388 | prevrev = None | |
382 |
|
389 | |||
383 | if deltamode == repository.CG_DELTAMODE_PREV or assumehaveparentrevisions: |
|
390 | if deltamode == repository.CG_DELTAMODE_PREV or assumehaveparentrevisions: | |
384 | prevrev = store.parentrevs(revs[0])[0] |
|
391 | prevrev = store.parentrevs(revs[0])[0] | |
385 |
|
392 | |||
386 | # Set of revs available to delta against. |
|
393 | # Set of revs available to delta against. | |
387 | available = set() |
|
394 | available = set() | |
388 |
|
395 | |||
389 | for rev in revs: |
|
396 | for rev in revs: | |
390 | if rev == nullrev: |
|
397 | if rev == nullrev: | |
391 | continue |
|
398 | continue | |
392 |
|
399 | |||
393 | node = fnode(rev) |
|
400 | node = fnode(rev) | |
394 | p1rev, p2rev = store.parentrevs(rev) |
|
401 | p1rev, p2rev = store.parentrevs(rev) | |
395 |
|
402 | |||
396 | if deltaparentfn: |
|
403 | if deltaparentfn: | |
397 | deltaparentrev = deltaparentfn(rev) |
|
404 | deltaparentrev = deltaparentfn(rev) | |
398 | else: |
|
405 | else: | |
399 | deltaparentrev = nullrev |
|
406 | deltaparentrev = nullrev | |
400 |
|
407 | |||
401 | # Forced delta against previous mode. |
|
408 | # Forced delta against previous mode. | |
402 | if deltamode == repository.CG_DELTAMODE_PREV: |
|
409 | if deltamode == repository.CG_DELTAMODE_PREV: | |
403 | baserev = prevrev |
|
410 | baserev = prevrev | |
404 |
|
411 | |||
405 | # We're instructed to send fulltext. Honor that. |
|
412 | # We're instructed to send fulltext. Honor that. | |
406 | elif deltamode == repository.CG_DELTAMODE_FULL: |
|
413 | elif deltamode == repository.CG_DELTAMODE_FULL: | |
407 | baserev = nullrev |
|
414 | baserev = nullrev | |
408 | # We're instructed to use p1. Honor that |
|
415 | # We're instructed to use p1. Honor that | |
409 | elif deltamode == repository.CG_DELTAMODE_P1: |
|
416 | elif deltamode == repository.CG_DELTAMODE_P1: | |
410 | baserev = p1rev |
|
417 | baserev = p1rev | |
411 |
|
418 | |||
412 | # There is a delta in storage. We try to use that because it |
|
419 | # There is a delta in storage. We try to use that because it | |
413 | # amounts to effectively copying data from storage and is |
|
420 | # amounts to effectively copying data from storage and is | |
414 | # therefore the fastest. |
|
421 | # therefore the fastest. | |
415 | elif deltaparentrev != nullrev: |
|
422 | elif deltaparentrev != nullrev: | |
416 | # Base revision was already emitted in this group. We can |
|
423 | # Base revision was already emitted in this group. We can | |
417 | # always safely use the delta. |
|
424 | # always safely use the delta. | |
418 | if deltaparentrev in available: |
|
425 | if deltaparentrev in available: | |
419 | baserev = deltaparentrev |
|
426 | baserev = deltaparentrev | |
420 |
|
427 | |||
421 | # Base revision is a parent that hasn't been emitted already. |
|
428 | # Base revision is a parent that hasn't been emitted already. | |
422 | # Use it if we can assume the receiver has the parent revision. |
|
429 | # Use it if we can assume the receiver has the parent revision. | |
423 | elif assumehaveparentrevisions and deltaparentrev in (p1rev, p2rev): |
|
430 | elif assumehaveparentrevisions and deltaparentrev in (p1rev, p2rev): | |
424 | baserev = deltaparentrev |
|
431 | baserev = deltaparentrev | |
425 |
|
432 | |||
426 | # No guarantee the receiver has the delta parent. Send delta |
|
433 | # No guarantee the receiver has the delta parent. Send delta | |
427 | # against last revision (if possible), which in the common case |
|
434 | # against last revision (if possible), which in the common case | |
428 | # should be similar enough to this revision that the delta is |
|
435 | # should be similar enough to this revision that the delta is | |
429 | # reasonable. |
|
436 | # reasonable. | |
430 | elif prevrev is not None: |
|
437 | elif prevrev is not None: | |
431 | baserev = prevrev |
|
438 | baserev = prevrev | |
432 | else: |
|
439 | else: | |
433 | baserev = nullrev |
|
440 | baserev = nullrev | |
434 |
|
441 | |||
435 | # Storage has a fulltext revision. |
|
442 | # Storage has a fulltext revision. | |
436 |
|
443 | |||
437 | # Let's use the previous revision, which is as good a guess as any. |
|
444 | # Let's use the previous revision, which is as good a guess as any. | |
438 | # There is definitely room to improve this logic. |
|
445 | # There is definitely room to improve this logic. | |
439 | elif prevrev is not None: |
|
446 | elif prevrev is not None: | |
440 | baserev = prevrev |
|
447 | baserev = prevrev | |
441 | else: |
|
448 | else: | |
442 | baserev = nullrev |
|
449 | baserev = nullrev | |
443 |
|
450 | |||
444 | # But we can't actually use our chosen delta base for whatever |
|
451 | # But we can't actually use our chosen delta base for whatever | |
445 | # reason. Reset to fulltext. |
|
452 | # reason. Reset to fulltext. | |
446 | if baserev != nullrev and (candeltafn and not candeltafn(baserev, rev)): |
|
453 | if baserev != nullrev and (candeltafn and not candeltafn(baserev, rev)): | |
447 | baserev = nullrev |
|
454 | baserev = nullrev | |
448 |
|
455 | |||
449 | revision = None |
|
456 | revision = None | |
450 | delta = None |
|
457 | delta = None | |
451 | baserevisionsize = None |
|
458 | baserevisionsize = None | |
452 |
|
459 | |||
453 | if revisiondata: |
|
460 | if revisiondata: | |
454 | if store.iscensored(baserev) or store.iscensored(rev): |
|
461 | if store.iscensored(baserev) or store.iscensored(rev): | |
455 | try: |
|
462 | try: | |
456 | revision = store.rawdata(node) |
|
463 | revision = store.rawdata(node) | |
457 | except error.CensoredNodeError as e: |
|
464 | except error.CensoredNodeError as e: | |
458 | revision = e.tombstone |
|
465 | revision = e.tombstone | |
459 |
|
466 | |||
460 | if baserev != nullrev: |
|
467 | if baserev != nullrev: | |
461 | if rawsizefn: |
|
468 | if rawsizefn: | |
462 | baserevisionsize = rawsizefn(baserev) |
|
469 | baserevisionsize = rawsizefn(baserev) | |
463 | else: |
|
470 | else: | |
464 | baserevisionsize = len(store.rawdata(baserev)) |
|
471 | baserevisionsize = len(store.rawdata(baserev)) | |
465 |
|
472 | |||
466 | elif ( |
|
473 | elif ( | |
467 | baserev == nullrev and deltamode != repository.CG_DELTAMODE_PREV |
|
474 | baserev == nullrev and deltamode != repository.CG_DELTAMODE_PREV | |
468 | ): |
|
475 | ): | |
469 | revision = store.rawdata(node) |
|
476 | revision = store.rawdata(node) | |
470 | available.add(rev) |
|
477 | available.add(rev) | |
471 | else: |
|
478 | else: | |
472 | if revdifffn: |
|
479 | if revdifffn: | |
473 | delta = revdifffn(baserev, rev) |
|
480 | delta = revdifffn(baserev, rev) | |
474 | else: |
|
481 | else: | |
475 | delta = mdiff.textdiff( |
|
482 | delta = mdiff.textdiff( | |
476 | store.rawdata(baserev), store.rawdata(rev) |
|
483 | store.rawdata(baserev), store.rawdata(rev) | |
477 | ) |
|
484 | ) | |
478 |
|
485 | |||
479 | available.add(rev) |
|
486 | available.add(rev) | |
480 |
|
487 | |||
481 | serialized_sidedata = None |
|
488 | serialized_sidedata = None | |
482 | sidedata_flags = (0, 0) |
|
489 | sidedata_flags = (0, 0) | |
483 | if sidedata_helpers: |
|
490 | if sidedata_helpers: | |
484 | try: |
|
491 | try: | |
485 | old_sidedata = store.sidedata(rev) |
|
492 | old_sidedata = store.sidedata(rev) | |
486 | except error.CensoredNodeError: |
|
493 | except error.CensoredNodeError: | |
487 | # skip any potential sidedata of the censored revision |
|
494 | # skip any potential sidedata of the censored revision | |
488 | sidedata = {} |
|
495 | sidedata = {} | |
489 | else: |
|
496 | else: | |
490 | sidedata, sidedata_flags = sidedatamod.run_sidedata_helpers( |
|
497 | sidedata, sidedata_flags = sidedatamod.run_sidedata_helpers( | |
491 | store=store, |
|
498 | store=store, | |
492 | sidedata_helpers=sidedata_helpers, |
|
499 | sidedata_helpers=sidedata_helpers, | |
493 | sidedata=old_sidedata, |
|
500 | sidedata=old_sidedata, | |
494 | rev=rev, |
|
501 | rev=rev, | |
495 | ) |
|
502 | ) | |
496 | if sidedata: |
|
503 | if sidedata: | |
497 | serialized_sidedata = sidedatamod.serialize_sidedata(sidedata) |
|
504 | serialized_sidedata = sidedatamod.serialize_sidedata(sidedata) | |
498 |
|
505 | |||
499 | flags = flagsfn(rev) if flagsfn else 0 |
|
506 | flags = flagsfn(rev) if flagsfn else 0 | |
500 | protocol_flags = 0 |
|
507 | protocol_flags = 0 | |
501 | if serialized_sidedata: |
|
508 | if serialized_sidedata: | |
502 | # Advertise that sidedata exists to the other side |
|
509 | # Advertise that sidedata exists to the other side | |
503 | protocol_flags |= CG_FLAG_SIDEDATA |
|
510 | protocol_flags |= CG_FLAG_SIDEDATA | |
504 | # Computers and removers can return flags to add and/or remove |
|
511 | # Computers and removers can return flags to add and/or remove | |
505 | flags = flags | sidedata_flags[0] & ~sidedata_flags[1] |
|
512 | flags = flags | sidedata_flags[0] & ~sidedata_flags[1] | |
506 |
|
513 | |||
507 | yield resultcls( |
|
514 | yield resultcls( | |
508 | node=node, |
|
515 | node=node, | |
509 | p1node=fnode(p1rev), |
|
516 | p1node=fnode(p1rev), | |
510 | p2node=fnode(p2rev), |
|
517 | p2node=fnode(p2rev), | |
511 | basenode=fnode(baserev), |
|
518 | basenode=fnode(baserev), | |
512 | flags=flags, |
|
519 | flags=flags, | |
513 | baserevisionsize=baserevisionsize, |
|
520 | baserevisionsize=baserevisionsize, | |
514 | revision=revision, |
|
521 | revision=revision, | |
515 | delta=delta, |
|
522 | delta=delta, | |
516 | sidedata=serialized_sidedata, |
|
523 | sidedata=serialized_sidedata, | |
517 | protocol_flags=protocol_flags, |
|
524 | protocol_flags=protocol_flags, | |
518 | ) |
|
525 | ) | |
519 |
|
526 | |||
520 | prevrev = rev |
|
527 | prevrev = rev | |
521 |
|
528 | |||
522 |
|
529 | |||
523 | def deltaiscensored(delta, baserev, baselenfn): |
|
530 | def deltaiscensored(delta, baserev, baselenfn): | |
524 | """Determine if a delta represents censored revision data. |
|
531 | """Determine if a delta represents censored revision data. | |
525 |
|
532 | |||
526 | ``baserev`` is the base revision this delta is encoded against. |
|
533 | ``baserev`` is the base revision this delta is encoded against. | |
527 | ``baselenfn`` is a callable receiving a revision number that resolves the |
|
534 | ``baselenfn`` is a callable receiving a revision number that resolves the | |
528 | length of the revision fulltext. |
|
535 | length of the revision fulltext. | |
529 |
|
536 | |||
530 | Returns a bool indicating if the result of the delta represents a censored |
|
537 | Returns a bool indicating if the result of the delta represents a censored | |
531 | revision. |
|
538 | revision. | |
532 | """ |
|
539 | """ | |
533 | # Fragile heuristic: unless new file meta keys are added alphabetically |
|
540 | # Fragile heuristic: unless new file meta keys are added alphabetically | |
534 | # preceding "censored", all censored revisions are prefixed by |
|
541 | # preceding "censored", all censored revisions are prefixed by | |
535 | # "\1\ncensored:". A delta producing such a censored revision must be a |
|
542 | # "\1\ncensored:". A delta producing such a censored revision must be a | |
536 | # full-replacement delta, so we inspect the first and only patch in the |
|
543 | # full-replacement delta, so we inspect the first and only patch in the | |
537 | # delta for this prefix. |
|
544 | # delta for this prefix. | |
538 | hlen = struct.calcsize(b">lll") |
|
545 | hlen = struct.calcsize(b">lll") | |
539 | if len(delta) <= hlen: |
|
546 | if len(delta) <= hlen: | |
540 | return False |
|
547 | return False | |
541 |
|
548 | |||
542 | oldlen = baselenfn(baserev) |
|
549 | oldlen = baselenfn(baserev) | |
543 | newlen = len(delta) - hlen |
|
550 | newlen = len(delta) - hlen | |
544 | if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen): |
|
551 | if delta[:hlen] != mdiff.replacediffheader(oldlen, newlen): | |
545 | return False |
|
552 | return False | |
546 |
|
553 | |||
547 | add = b"\1\ncensored:" |
|
554 | add = b"\1\ncensored:" | |
548 | addlen = len(add) |
|
555 | addlen = len(add) | |
549 | return newlen >= addlen and delta[hlen : hlen + addlen] == add |
|
556 | return newlen >= addlen and delta[hlen : hlen + addlen] == add |
General Comments 0
You need to be logged in to leave comments.
Login now