Show More
@@ -587,6 +587,7 b' class sqlitefilestore(object):' | |||||
587 | revisiondata=False, |
|
587 | revisiondata=False, | |
588 | assumehaveparentrevisions=False, |
|
588 | assumehaveparentrevisions=False, | |
589 | deltamode=repository.CG_DELTAMODE_STD, |
|
589 | deltamode=repository.CG_DELTAMODE_STD, | |
|
590 | sidedata_helpers=None, | |||
590 | ): |
|
591 | ): | |
591 | if nodesorder not in (b'nodes', b'storage', b'linear', None): |
|
592 | if nodesorder not in (b'nodes', b'storage', b'linear', None): | |
592 | raise error.ProgrammingError( |
|
593 | raise error.ProgrammingError( | |
@@ -625,6 +626,7 b' class sqlitefilestore(object):' | |||||
625 | revisiondata=revisiondata, |
|
626 | revisiondata=revisiondata, | |
626 | assumehaveparentrevisions=assumehaveparentrevisions, |
|
627 | assumehaveparentrevisions=assumehaveparentrevisions, | |
627 | deltamode=deltamode, |
|
628 | deltamode=deltamode, | |
|
629 | sidedata_helpers=sidedata_helpers, | |||
628 | ): |
|
630 | ): | |
629 |
|
631 | |||
630 | yield delta |
|
632 | yield delta |
@@ -272,6 +272,7 b' class cg1unpacker(object):' | |||||
272 | url, |
|
272 | url, | |
273 | targetphase=phases.draft, |
|
273 | targetphase=phases.draft, | |
274 | expectedtotal=None, |
|
274 | expectedtotal=None, | |
|
275 | sidedata_categories=None, | |||
275 | ): |
|
276 | ): | |
276 | """Add the changegroup returned by source.read() to this repo. |
|
277 | """Add the changegroup returned by source.read() to this repo. | |
277 | srctype is a string like 'push', 'pull', or 'unbundle'. url is |
|
278 | srctype is a string like 'push', 'pull', or 'unbundle'. url is | |
@@ -282,9 +283,23 b' class cg1unpacker(object):' | |||||
282 | - more heads than before: 1+added heads (2..n) |
|
283 | - more heads than before: 1+added heads (2..n) | |
283 | - fewer heads than before: -1-removed heads (-2..-n) |
|
284 | - fewer heads than before: -1-removed heads (-2..-n) | |
284 | - number of heads stays the same: 1 |
|
285 | - number of heads stays the same: 1 | |
|
286 | ||||
|
287 | `sidedata_categories` is an optional set of the remote's sidedata wanted | |||
|
288 | categories. | |||
285 | """ |
|
289 | """ | |
286 | repo = repo.unfiltered() |
|
290 | repo = repo.unfiltered() | |
287 |
|
291 | |||
|
292 | # Only useful if we're adding sidedata categories. If both peers have | |||
|
293 | # the same categories, then we simply don't do anything. | |||
|
294 | if self.version == b'04' and srctype == b'pull': | |||
|
295 | sidedata_helpers = get_sidedata_helpers( | |||
|
296 | repo, | |||
|
297 | sidedata_categories or set(), | |||
|
298 | pull=True, | |||
|
299 | ) | |||
|
300 | else: | |||
|
301 | sidedata_helpers = None | |||
|
302 | ||||
288 | def csmap(x): |
|
303 | def csmap(x): | |
289 | repo.ui.debug(b"add changeset %s\n" % short(x)) |
|
304 | repo.ui.debug(b"add changeset %s\n" % short(x)) | |
290 | return len(cl) |
|
305 | return len(cl) | |
@@ -749,6 +764,7 b' def deltagroup(' | |||||
749 | clrevtolocalrev=None, |
|
764 | clrevtolocalrev=None, | |
750 | fullclnodes=None, |
|
765 | fullclnodes=None, | |
751 | precomputedellipsis=None, |
|
766 | precomputedellipsis=None, | |
|
767 | sidedata_helpers=None, | |||
752 | ): |
|
768 | ): | |
753 | """Calculate deltas for a set of revisions. |
|
769 | """Calculate deltas for a set of revisions. | |
754 |
|
770 | |||
@@ -756,6 +772,8 b' def deltagroup(' | |||||
756 |
|
772 | |||
757 | If topic is not None, progress detail will be generated using this |
|
773 | If topic is not None, progress detail will be generated using this | |
758 | topic name (e.g. changesets, manifests, etc). |
|
774 | topic name (e.g. changesets, manifests, etc). | |
|
775 | ||||
|
776 | See `storageutil.emitrevisions` for the doc on `sidedata_helpers`. | |||
759 | """ |
|
777 | """ | |
760 | if not nodes: |
|
778 | if not nodes: | |
761 | return |
|
779 | return | |
@@ -854,6 +872,7 b' def deltagroup(' | |||||
854 | revisiondata=True, |
|
872 | revisiondata=True, | |
855 | assumehaveparentrevisions=not ellipses, |
|
873 | assumehaveparentrevisions=not ellipses, | |
856 | deltamode=deltamode, |
|
874 | deltamode=deltamode, | |
|
875 | sidedata_helpers=sidedata_helpers, | |||
857 | ) |
|
876 | ) | |
858 |
|
877 | |||
859 | for i, revision in enumerate(revisions): |
|
878 | for i, revision in enumerate(revisions): | |
@@ -974,8 +993,21 b' class cgpacker(object):' | |||||
974 | self._verbosenote(_(b'uncompressed size of bundle content:\n')) |
|
993 | self._verbosenote(_(b'uncompressed size of bundle content:\n')) | |
975 | size = 0 |
|
994 | size = 0 | |
976 |
|
995 | |||
|
996 | sidedata_helpers = None | |||
|
997 | if self.version == b'04': | |||
|
998 | remote_sidedata = self._remote_sidedata | |||
|
999 | if source == b'strip': | |||
|
1000 | # We're our own remote when stripping, get the no-op helpers | |||
|
1001 | # TODO a better approach would be for the strip bundle to | |||
|
1002 | # correctly advertise its sidedata categories directly. | |||
|
1003 | remote_sidedata = repo._wanted_sidedata | |||
|
1004 | sidedata_helpers = get_sidedata_helpers(repo, remote_sidedata) | |||
|
1005 | ||||
977 | clstate, deltas = self._generatechangelog( |
|
1006 | clstate, deltas = self._generatechangelog( | |
978 | cl, clnodes, generate=changelog |
|
1007 | cl, | |
|
1008 | clnodes, | |||
|
1009 | generate=changelog, | |||
|
1010 | sidedata_helpers=sidedata_helpers, | |||
979 | ) |
|
1011 | ) | |
980 | for delta in deltas: |
|
1012 | for delta in deltas: | |
981 | for chunk in _revisiondeltatochunks(delta, self._builddeltaheader): |
|
1013 | for chunk in _revisiondeltatochunks(delta, self._builddeltaheader): | |
@@ -1023,6 +1055,7 b' class cgpacker(object):' | |||||
1023 | fnodes, |
|
1055 | fnodes, | |
1024 | source, |
|
1056 | source, | |
1025 | clstate[b'clrevtomanifestrev'], |
|
1057 | clstate[b'clrevtomanifestrev'], | |
|
1058 | sidedata_helpers=sidedata_helpers, | |||
1026 | ) |
|
1059 | ) | |
1027 |
|
1060 | |||
1028 | for tree, deltas in it: |
|
1061 | for tree, deltas in it: | |
@@ -1063,6 +1096,7 b' class cgpacker(object):' | |||||
1063 | fastpathlinkrev, |
|
1096 | fastpathlinkrev, | |
1064 | fnodes, |
|
1097 | fnodes, | |
1065 | clrevs, |
|
1098 | clrevs, | |
|
1099 | sidedata_helpers=sidedata_helpers, | |||
1066 | ) |
|
1100 | ) | |
1067 |
|
1101 | |||
1068 | for path, deltas in it: |
|
1102 | for path, deltas in it: | |
@@ -1087,7 +1121,9 b' class cgpacker(object):' | |||||
1087 | if clnodes: |
|
1121 | if clnodes: | |
1088 | repo.hook(b'outgoing', node=hex(clnodes[0]), source=source) |
|
1122 | repo.hook(b'outgoing', node=hex(clnodes[0]), source=source) | |
1089 |
|
1123 | |||
1090 |
def _generatechangelog( |
|
1124 | def _generatechangelog( | |
|
1125 | self, cl, nodes, generate=True, sidedata_helpers=None | |||
|
1126 | ): | |||
1091 | """Generate data for changelog chunks. |
|
1127 | """Generate data for changelog chunks. | |
1092 |
|
1128 | |||
1093 | Returns a 2-tuple of a dict containing state and an iterable of |
|
1129 | Returns a 2-tuple of a dict containing state and an iterable of | |
@@ -1096,6 +1132,8 b' class cgpacker(object):' | |||||
1096 |
|
1132 | |||
1097 | if generate is False, the state will be fully populated and no chunk |
|
1133 | if generate is False, the state will be fully populated and no chunk | |
1098 | stream will be yielded |
|
1134 | stream will be yielded | |
|
1135 | ||||
|
1136 | See `storageutil.emitrevisions` for the doc on `sidedata_helpers`. | |||
1099 | """ |
|
1137 | """ | |
1100 | clrevorder = {} |
|
1138 | clrevorder = {} | |
1101 | manifests = {} |
|
1139 | manifests = {} | |
@@ -1179,6 +1217,7 b' class cgpacker(object):' | |||||
1179 | clrevtolocalrev={}, |
|
1217 | clrevtolocalrev={}, | |
1180 | fullclnodes=self._fullclnodes, |
|
1218 | fullclnodes=self._fullclnodes, | |
1181 | precomputedellipsis=self._precomputedellipsis, |
|
1219 | precomputedellipsis=self._precomputedellipsis, | |
|
1220 | sidedata_helpers=sidedata_helpers, | |||
1182 | ) |
|
1221 | ) | |
1183 |
|
1222 | |||
1184 | return state, gen |
|
1223 | return state, gen | |
@@ -1192,11 +1231,14 b' class cgpacker(object):' | |||||
1192 | fnodes, |
|
1231 | fnodes, | |
1193 | source, |
|
1232 | source, | |
1194 | clrevtolocalrev, |
|
1233 | clrevtolocalrev, | |
|
1234 | sidedata_helpers=None, | |||
1195 | ): |
|
1235 | ): | |
1196 | """Returns an iterator of changegroup chunks containing manifests. |
|
1236 | """Returns an iterator of changegroup chunks containing manifests. | |
1197 |
|
1237 | |||
1198 | `source` is unused here, but is used by extensions like remotefilelog to |
|
1238 | `source` is unused here, but is used by extensions like remotefilelog to | |
1199 | change what is sent based in pulls vs pushes, etc. |
|
1239 | change what is sent based in pulls vs pushes, etc. | |
|
1240 | ||||
|
1241 | See `storageutil.emitrevisions` for the doc on `sidedata_helpers`. | |||
1200 | """ |
|
1242 | """ | |
1201 | repo = self._repo |
|
1243 | repo = self._repo | |
1202 | mfl = repo.manifestlog |
|
1244 | mfl = repo.manifestlog | |
@@ -1285,6 +1327,7 b' class cgpacker(object):' | |||||
1285 | clrevtolocalrev=clrevtolocalrev, |
|
1327 | clrevtolocalrev=clrevtolocalrev, | |
1286 | fullclnodes=self._fullclnodes, |
|
1328 | fullclnodes=self._fullclnodes, | |
1287 | precomputedellipsis=self._precomputedellipsis, |
|
1329 | precomputedellipsis=self._precomputedellipsis, | |
|
1330 | sidedata_helpers=sidedata_helpers, | |||
1288 | ) |
|
1331 | ) | |
1289 |
|
1332 | |||
1290 | if not self._oldmatcher.visitdir(store.tree[:-1]): |
|
1333 | if not self._oldmatcher.visitdir(store.tree[:-1]): | |
@@ -1323,6 +1366,7 b' class cgpacker(object):' | |||||
1323 | fastpathlinkrev, |
|
1366 | fastpathlinkrev, | |
1324 | fnodes, |
|
1367 | fnodes, | |
1325 | clrevs, |
|
1368 | clrevs, | |
|
1369 | sidedata_helpers=None, | |||
1326 | ): |
|
1370 | ): | |
1327 | changedfiles = [ |
|
1371 | changedfiles = [ | |
1328 | f |
|
1372 | f | |
@@ -1417,6 +1461,7 b' class cgpacker(object):' | |||||
1417 | clrevtolocalrev=clrevtolocalrev, |
|
1461 | clrevtolocalrev=clrevtolocalrev, | |
1418 | fullclnodes=self._fullclnodes, |
|
1462 | fullclnodes=self._fullclnodes, | |
1419 | precomputedellipsis=self._precomputedellipsis, |
|
1463 | precomputedellipsis=self._precomputedellipsis, | |
|
1464 | sidedata_helpers=sidedata_helpers, | |||
1420 | ) |
|
1465 | ) | |
1421 |
|
1466 | |||
1422 | yield fname, deltas |
|
1467 | yield fname, deltas | |
@@ -1792,3 +1837,25 b' def _addchangegroupfiles(repo, source, r' | |||||
1792 | ) |
|
1837 | ) | |
1793 |
|
1838 | |||
1794 | return revisions, files |
|
1839 | return revisions, files | |
|
1840 | ||||
|
1841 | ||||
|
1842 | def get_sidedata_helpers(repo, remote_sd_categories, pull=False): | |||
|
1843 | # Computers for computing sidedata on-the-fly | |||
|
1844 | sd_computers = collections.defaultdict(list) | |||
|
1845 | # Computers for categories to remove from sidedata | |||
|
1846 | sd_removers = collections.defaultdict(list) | |||
|
1847 | ||||
|
1848 | to_generate = remote_sd_categories - repo._wanted_sidedata | |||
|
1849 | to_remove = repo._wanted_sidedata - remote_sd_categories | |||
|
1850 | if pull: | |||
|
1851 | to_generate, to_remove = to_remove, to_generate | |||
|
1852 | ||||
|
1853 | for revlog_kind, computers in repo._sidedata_computers.items(): | |||
|
1854 | for category, computer in computers.items(): | |||
|
1855 | if category in to_generate: | |||
|
1856 | sd_computers[revlog_kind].append(computer) | |||
|
1857 | if category in to_remove: | |||
|
1858 | sd_removers[revlog_kind].append(computer) | |||
|
1859 | ||||
|
1860 | sidedata_helpers = (repo, sd_computers, sd_removers) | |||
|
1861 | return sidedata_helpers |
@@ -103,6 +103,7 b' class filelog(object):' | |||||
103 | revisiondata=False, |
|
103 | revisiondata=False, | |
104 | assumehaveparentrevisions=False, |
|
104 | assumehaveparentrevisions=False, | |
105 | deltamode=repository.CG_DELTAMODE_STD, |
|
105 | deltamode=repository.CG_DELTAMODE_STD, | |
|
106 | sidedata_helpers=None, | |||
106 | ): |
|
107 | ): | |
107 | return self._revlog.emitrevisions( |
|
108 | return self._revlog.emitrevisions( | |
108 | nodes, |
|
109 | nodes, | |
@@ -110,6 +111,7 b' class filelog(object):' | |||||
110 | revisiondata=revisiondata, |
|
111 | revisiondata=revisiondata, | |
111 | assumehaveparentrevisions=assumehaveparentrevisions, |
|
112 | assumehaveparentrevisions=assumehaveparentrevisions, | |
112 | deltamode=deltamode, |
|
113 | deltamode=deltamode, | |
|
114 | sidedata_helpers=sidedata_helpers, | |||
113 | ) |
|
115 | ) | |
114 |
|
116 | |||
115 | def addrevision( |
|
117 | def addrevision( |
@@ -1826,6 +1826,7 b' class manifestrevlog(object):' | |||||
1826 | revisiondata=False, |
|
1826 | revisiondata=False, | |
1827 | assumehaveparentrevisions=False, |
|
1827 | assumehaveparentrevisions=False, | |
1828 | deltamode=repository.CG_DELTAMODE_STD, |
|
1828 | deltamode=repository.CG_DELTAMODE_STD, | |
|
1829 | sidedata_helpers=None, | |||
1829 | ): |
|
1830 | ): | |
1830 | return self._revlog.emitrevisions( |
|
1831 | return self._revlog.emitrevisions( | |
1831 | nodes, |
|
1832 | nodes, | |
@@ -1833,6 +1834,7 b' class manifestrevlog(object):' | |||||
1833 | revisiondata=revisiondata, |
|
1834 | revisiondata=revisiondata, | |
1834 | assumehaveparentrevisions=assumehaveparentrevisions, |
|
1835 | assumehaveparentrevisions=assumehaveparentrevisions, | |
1835 | deltamode=deltamode, |
|
1836 | deltamode=deltamode, | |
|
1837 | sidedata_helpers=sidedata_helpers, | |||
1836 | ) |
|
1838 | ) | |
1837 |
|
1839 | |||
1838 | def addgroup( |
|
1840 | def addgroup( |
@@ -2733,6 +2733,7 b' class revlog(object):' | |||||
2733 | revisiondata=False, |
|
2733 | revisiondata=False, | |
2734 | assumehaveparentrevisions=False, |
|
2734 | assumehaveparentrevisions=False, | |
2735 | deltamode=repository.CG_DELTAMODE_STD, |
|
2735 | deltamode=repository.CG_DELTAMODE_STD, | |
|
2736 | sidedata_helpers=None, | |||
2736 | ): |
|
2737 | ): | |
2737 | if nodesorder not in (b'nodes', b'storage', b'linear', None): |
|
2738 | if nodesorder not in (b'nodes', b'storage', b'linear', None): | |
2738 | raise error.ProgrammingError( |
|
2739 | raise error.ProgrammingError( | |
@@ -2761,6 +2762,7 b' class revlog(object):' | |||||
2761 | deltamode=deltamode, |
|
2762 | deltamode=deltamode, | |
2762 | revisiondata=revisiondata, |
|
2763 | revisiondata=revisiondata, | |
2763 | assumehaveparentrevisions=assumehaveparentrevisions, |
|
2764 | assumehaveparentrevisions=assumehaveparentrevisions, | |
|
2765 | sidedata_helpers=sidedata_helpers, | |||
2764 | ) |
|
2766 | ) | |
2765 |
|
2767 | |||
2766 | DELTAREUSEALWAYS = b'always' |
|
2768 | DELTAREUSEALWAYS = b'always' |
@@ -23,6 +23,7 b' from .. import (' | |||||
23 | pycompat, |
|
23 | pycompat, | |
24 | ) |
|
24 | ) | |
25 | from ..interfaces import repository |
|
25 | from ..interfaces import repository | |
|
26 | from ..revlogutils import sidedata as sidedatamod | |||
26 | from ..utils import hashutil |
|
27 | from ..utils import hashutil | |
27 |
|
28 | |||
28 | _nullhash = hashutil.sha1(nullid) |
|
29 | _nullhash = hashutil.sha1(nullid) | |
@@ -294,6 +295,7 b' def emitrevisions(' | |||||
294 | deltamode=repository.CG_DELTAMODE_STD, |
|
295 | deltamode=repository.CG_DELTAMODE_STD, | |
295 | revisiondata=False, |
|
296 | revisiondata=False, | |
296 | assumehaveparentrevisions=False, |
|
297 | assumehaveparentrevisions=False, | |
|
298 | sidedata_helpers=None, | |||
297 | ): |
|
299 | ): | |
298 | """Generic implementation of ifiledata.emitrevisions(). |
|
300 | """Generic implementation of ifiledata.emitrevisions(). | |
299 |
|
301 | |||
@@ -356,6 +358,21 b' def emitrevisions(' | |||||
356 | ``nodesorder`` |
|
358 | ``nodesorder`` | |
357 | ``revisiondata`` |
|
359 | ``revisiondata`` | |
358 | ``assumehaveparentrevisions`` |
|
360 | ``assumehaveparentrevisions`` | |
|
361 | ``sidedata_helpers`` (optional) | |||
|
362 | If not None, means that sidedata should be included. | |||
|
363 | A dictionary of revlog type to tuples of `(repo, computers, removers)`: | |||
|
364 | * `repo` is used as an argument for computers | |||
|
365 | * `computers` is a list of `(category, (keys, computer)` that | |||
|
366 | compute the missing sidedata categories that were asked: | |||
|
367 | * `category` is the sidedata category | |||
|
368 | * `keys` are the sidedata keys to be affected | |||
|
369 | * `computer` is the function `(repo, store, rev, sidedata)` that | |||
|
370 | returns a new sidedata dict. | |||
|
371 | * `removers` will remove the keys corresponding to the categories | |||
|
372 | that are present, but not needed. | |||
|
373 | If both `computers` and `removers` are empty, sidedata are simply not | |||
|
374 | transformed. | |||
|
375 | Revlog types are `changelog`, `manifest` or `filelog`. | |||
359 | """ |
|
376 | """ | |
360 |
|
377 | |||
361 | fnode = store.node |
|
378 | fnode = store.node | |
@@ -469,6 +486,17 b' def emitrevisions(' | |||||
469 |
|
486 | |||
470 | available.add(rev) |
|
487 | available.add(rev) | |
471 |
|
488 | |||
|
489 | sidedata = None | |||
|
490 | if sidedata_helpers: | |||
|
491 | sidedata = store.sidedata(rev) | |||
|
492 | sidedata = run_sidedata_helpers( | |||
|
493 | store=store, | |||
|
494 | sidedata_helpers=sidedata_helpers, | |||
|
495 | sidedata=sidedata, | |||
|
496 | rev=rev, | |||
|
497 | ) | |||
|
498 | sidedata = sidedatamod.serialize_sidedata(sidedata) | |||
|
499 | ||||
472 | yield resultcls( |
|
500 | yield resultcls( | |
473 | node=node, |
|
501 | node=node, | |
474 | p1node=fnode(p1rev), |
|
502 | p1node=fnode(p1rev), | |
@@ -484,6 +512,25 b' def emitrevisions(' | |||||
484 | prevrev = rev |
|
512 | prevrev = rev | |
485 |
|
513 | |||
486 |
|
514 | |||
|
515 | def run_sidedata_helpers(store, sidedata_helpers, sidedata, rev): | |||
|
516 | """Returns the sidedata for the given revision after running through | |||
|
517 | the given helpers. | |||
|
518 | - `store`: the revlog this applies to (changelog, manifest, or filelog | |||
|
519 | instance) | |||
|
520 | - `sidedata_helpers`: see `storageutil.emitrevisions` | |||
|
521 | - `sidedata`: previous sidedata at the given rev, if any | |||
|
522 | - `rev`: affected rev of `store` | |||
|
523 | """ | |||
|
524 | repo, sd_computers, sd_removers = sidedata_helpers | |||
|
525 | kind = store.revlog_kind | |||
|
526 | for _keys, sd_computer in sd_computers.get(kind, []): | |||
|
527 | sidedata = sd_computer(repo, store, rev, sidedata) | |||
|
528 | for keys, _computer in sd_removers.get(kind, []): | |||
|
529 | for key in keys: | |||
|
530 | sidedata.pop(key, None) | |||
|
531 | return sidedata | |||
|
532 | ||||
|
533 | ||||
487 | def deltaiscensored(delta, baserev, baselenfn): |
|
534 | def deltaiscensored(delta, baserev, baselenfn): | |
488 | """Determine if a delta represents censored revision data. |
|
535 | """Determine if a delta represents censored revision data. | |
489 |
|
536 |
@@ -446,6 +446,7 b' class filestorage(object):' | |||||
446 | revisiondata=False, |
|
446 | revisiondata=False, | |
447 | assumehaveparentrevisions=False, |
|
447 | assumehaveparentrevisions=False, | |
448 | deltamode=repository.CG_DELTAMODE_STD, |
|
448 | deltamode=repository.CG_DELTAMODE_STD, | |
|
449 | sidedata_helpers=None, | |||
449 | ): |
|
450 | ): | |
450 | # TODO this will probably break on some ordering options. |
|
451 | # TODO this will probably break on some ordering options. | |
451 | nodes = [n for n in nodes if n != nullid] |
|
452 | nodes = [n for n in nodes if n != nullid] | |
@@ -459,6 +460,7 b' class filestorage(object):' | |||||
459 | revisiondata=revisiondata, |
|
460 | revisiondata=revisiondata, | |
460 | assumehaveparentrevisions=assumehaveparentrevisions, |
|
461 | assumehaveparentrevisions=assumehaveparentrevisions, | |
461 | deltamode=deltamode, |
|
462 | deltamode=deltamode, | |
|
463 | sidedata_helpers=sidedata_helpers, | |||
462 | ): |
|
464 | ): | |
463 | yield delta |
|
465 | yield delta | |
464 |
|
466 |
General Comments 0
You need to be logged in to leave comments.
Login now