##// END OF EJS Templates
exchangev2: support for calling rawstorefiledata to retrieve raw files...
Gregory Szorc -
r40366:00a4cd36 default
parent child Browse files
Show More
@@ -29,6 +29,18 b' def pull(pullop):'
29 """Pull using wire protocol version 2."""
29 """Pull using wire protocol version 2."""
30 repo = pullop.repo
30 repo = pullop.repo
31 remote = pullop.remote
31 remote = pullop.remote
32
33 usingrawchangelogandmanifest = _checkuserawstorefiledata(pullop)
34
35 # If this is a clone and it was requested to perform a "stream clone",
36 # we obtain the raw files data from the remote then fall back to an
37 # incremental pull. This is somewhat hacky and is not nearly robust enough
38 # for long-term usage.
39 if usingrawchangelogandmanifest:
40 with repo.transaction('clone'):
41 _fetchrawstorefiles(repo, remote)
42 repo.invalidate(clearfilecache=True)
43
32 tr = pullop.trmanager.transaction()
44 tr = pullop.trmanager.transaction()
33
45
34 # We don't use the repo's narrow matcher here because the patterns passed
46 # We don't use the repo's narrow matcher here because the patterns passed
@@ -79,11 +91,122 b' def pull(pullop):'
79
91
80 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
92 manres = _fetchmanifests(repo, tr, remote, csetres['manifestnodes'])
81
93
94 # If obtaining the raw store files, we need to scan the full repo to
95 # derive all the changesets, manifests, and linkrevs.
96 if usingrawchangelogandmanifest:
97 csetsforfiles = []
98 mnodesforfiles = []
99 manifestlinkrevs = {}
100
101 for rev in repo:
102 ctx = repo[rev]
103 mnode = ctx.manifestnode()
104
105 csetsforfiles.append(ctx.node())
106 mnodesforfiles.append(mnode)
107 manifestlinkrevs[mnode] = rev
108
109 else:
110 csetsforfiles = csetres['added']
111 mnodesforfiles = manres['added']
112 manifestlinkrevs = manres['linkrevs']
113
82 # Find all file nodes referenced by added manifests and fetch those
114 # Find all file nodes referenced by added manifests and fetch those
83 # revisions.
115 # revisions.
84 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, manres['added'])
116 fnodes = _derivefilesfrommanifests(repo, narrowmatcher, mnodesforfiles)
85 _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetres['added'],
117 _fetchfilesfromcsets(repo, tr, remote, pathfilter, fnodes, csetsforfiles,
86 manres['linkrevs'])
118 manifestlinkrevs)
119
120 def _checkuserawstorefiledata(pullop):
121 """Check whether we should use rawstorefiledata command to retrieve data."""
122
123 repo = pullop.repo
124 remote = pullop.remote
125
126 # Command to obtain raw store data isn't available.
127 if b'rawstorefiledata' not in remote.apidescriptor[b'commands']:
128 return False
129
130 # Only honor if user requested stream clone operation.
131 if not pullop.streamclonerequested:
132 return False
133
134 # Only works on empty repos.
135 if len(repo):
136 return False
137
138 # TODO This is super hacky. There needs to be a storage API for this. We
139 # also need to check for compatibility with the remote.
140 if b'revlogv1' not in repo.requirements:
141 return False
142
143 return True
144
145 def _fetchrawstorefiles(repo, remote):
146 with remote.commandexecutor() as e:
147 objs = e.callcommand(b'rawstorefiledata', {
148 b'files': [b'changelog', b'manifestlog'],
149 }).result()
150
151 # First object is a summary of files data that follows.
152 overall = next(objs)
153
154 progress = repo.ui.makeprogress(_('clone'), total=overall[b'totalsize'],
155 unit=_('bytes'))
156 with progress:
157 progress.update(0)
158
159 # Next are pairs of file metadata, data.
160 while True:
161 try:
162 filemeta = next(objs)
163 except StopIteration:
164 break
165
166 for k in (b'location', b'path', b'size'):
167 if k not in filemeta:
168 raise error.Abort(_(b'remote file data missing key: %s')
169 % k)
170
171 if filemeta[b'location'] == b'store':
172 vfs = repo.svfs
173 else:
174 raise error.Abort(_(b'invalid location for raw file data: '
175 b'%s') % filemeta[b'location'])
176
177 bytesremaining = filemeta[b'size']
178
179 with vfs.open(filemeta[b'path'], b'wb') as fh:
180 while True:
181 try:
182 chunk = next(objs)
183 except StopIteration:
184 break
185
186 bytesremaining -= len(chunk)
187
188 if bytesremaining < 0:
189 raise error.Abort(_(
190 b'received invalid number of bytes for file '
191 b'data; expected %d, got extra') %
192 filemeta[b'size'])
193
194 progress.increment(step=len(chunk))
195 fh.write(chunk)
196
197 try:
198 if chunk.islast:
199 break
200 except AttributeError:
201 raise error.Abort(_(
202 b'did not receive indefinite length bytestring '
203 b'for file data'))
204
205 if bytesremaining:
206 raise error.Abort(_(b'received invalid number of bytes for'
207 b'file data; expected %d got %d') %
208 (filemeta[b'size'],
209 filemeta[b'size'] - bytesremaining))
87
210
88 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
211 def _pullchangesetdiscovery(repo, remote, heads, abortwhenunrelated=True):
89 """Determine which changesets need to be pulled."""
212 """Determine which changesets need to be pulled."""
@@ -963,3 +963,185 b' Mixing --include and --exclude works'
963 client-narrow-2/.hg/store/00manifest.i
963 client-narrow-2/.hg/store/00manifest.i
964 client-narrow-2/.hg/store/data/dir0/d.i
964 client-narrow-2/.hg/store/data/dir0/d.i
965 #endif
965 #endif
966
967 --stream will use rawfiledata to transfer changelog and manifestlog, then
968 fall through to get files data
969
970 $ hg --debug clone --stream -U http://localhost:$HGPORT client-stream-0
971 using http://localhost:$HGPORT/
972 sending capabilities command
973 sending 1 commands
974 sending command rawstorefiledata: {
975 'files': [
976 'changelog',
977 'manifestlog'
978 ]
979 }
980 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
981 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
982 received frame(size=1275; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
983 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
984 updating the branch cache
985 query 1; heads
986 sending 2 commands
987 sending command heads: {}
988 sending command known: {
989 'nodes': [
990 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
991 ]
992 }
993 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
994 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
995 received frame(size=22; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
996 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
997 received frame(size=11; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
998 received frame(size=2; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
999 received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
1000 searching for changes
1001 all remote heads known locally
1002 sending 1 commands
1003 sending command changesetdata: {
1004 'fields': set([
1005 'bookmarks',
1006 'parents',
1007 'phase',
1008 'revision'
1009 ]),
1010 'revisions': [
1011 {
1012 'heads': [
1013 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1014 ],
1015 'roots': [
1016 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1017 ],
1018 'type': 'changesetdagrange'
1019 }
1020 ]
1021 }
1022 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
1023 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1024 received frame(size=13; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1025 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
1026 checking for updated bookmarks
1027 sending 1 commands
1028 sending command filesdata: {
1029 'fields': set([
1030 'parents',
1031 'revision'
1032 ]),
1033 'haveparents': True,
1034 'revisions': [
1035 {
1036 'nodes': [
1037 '3\x90\xef\x85\x00s\xfb\xc2\xf0\xdf\xff"D4,\x8e\x92)\x01:',
1038 '\xb7\t8\x08\x92\xb1\x93\xc1\t\x1d:\x81\x7fp`R\xe3F\x82\x1b',
1039 'G\xfe\x01*\xb27\xa8\xc7\xfc\x0cx\xf9\xf2mXf\xee\xf3\xf8%',
1040 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1041 ],
1042 'type': 'changesetexplicit'
1043 }
1044 ]
1045 }
1046 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
1047 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1048 received frame(size=1133; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1049 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
1050 (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob)
1051
1052 --stream + --include/--exclude will only obtain some files
1053
1054 $ hg --debug --config extensions.pullext=$TESTDIR/pullext.py clone --stream --include dir0/ -U http://localhost:$HGPORT client-stream-2
1055 using http://localhost:$HGPORT/
1056 sending capabilities command
1057 sending 1 commands
1058 sending command rawstorefiledata: {
1059 'files': [
1060 'changelog',
1061 'manifestlog'
1062 ]
1063 }
1064 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
1065 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1066 received frame(size=1275; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1067 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
1068 updating the branch cache
1069 query 1; heads
1070 sending 2 commands
1071 sending command heads: {}
1072 sending command known: {
1073 'nodes': [
1074 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1075 ]
1076 }
1077 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
1078 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1079 received frame(size=22; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1080 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
1081 received frame(size=11; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1082 received frame(size=2; request=3; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1083 received frame(size=0; request=3; stream=2; streamflags=; type=command-response; flags=eos)
1084 searching for changes
1085 all remote heads known locally
1086 sending 1 commands
1087 sending command changesetdata: {
1088 'fields': set([
1089 'bookmarks',
1090 'parents',
1091 'phase',
1092 'revision'
1093 ]),
1094 'revisions': [
1095 {
1096 'heads': [
1097 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1098 ],
1099 'roots': [
1100 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1101 ],
1102 'type': 'changesetdagrange'
1103 }
1104 ]
1105 }
1106 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
1107 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1108 received frame(size=13; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1109 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
1110 checking for updated bookmarks
1111 sending 1 commands
1112 sending command filesdata: {
1113 'fields': set([
1114 'parents',
1115 'revision'
1116 ]),
1117 'haveparents': True,
1118 'pathfilter': {
1119 'include': [
1120 'path:dir0'
1121 ]
1122 },
1123 'revisions': [
1124 {
1125 'nodes': [
1126 '3\x90\xef\x85\x00s\xfb\xc2\xf0\xdf\xff"D4,\x8e\x92)\x01:',
1127 '\xb7\t8\x08\x92\xb1\x93\xc1\t\x1d:\x81\x7fp`R\xe3F\x82\x1b',
1128 'G\xfe\x01*\xb27\xa8\xc7\xfc\x0cx\xf9\xf2mXf\xee\xf3\xf8%',
1129 '\x97v_\xc3\xcdbO\xd1\xfa\x01v\x93,!\xff\xd1j\xdfC.'
1130 ],
1131 'type': 'changesetexplicit'
1132 }
1133 ]
1134 }
1135 received frame(size=9; request=1; stream=2; streamflags=stream-begin; type=stream-settings; flags=eos)
1136 received frame(size=11; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1137 received frame(size=449; request=1; stream=2; streamflags=encoded; type=command-response; flags=continuation)
1138 received frame(size=0; request=1; stream=2; streamflags=; type=command-response; flags=eos)
1139 (sent 5 HTTP requests and * bytes; received * bytes in responses) (glob)
1140
1141 #if reporevlogstore
1142 $ find client-stream-2/.hg/store -type f -name '*.i' | sort
1143 client-stream-2/.hg/store/00changelog.i
1144 client-stream-2/.hg/store/00manifest.i
1145 client-stream-2/.hg/store/data/dir0/c.i
1146 client-stream-2/.hg/store/data/dir0/d.i
1147 #endif
General Comments 0
You need to be logged in to leave comments. Login now