Show More
@@ -1,528 +1,548 b'' | |||||
1 | # upgrade.py - functions for in place upgrade of Mercurial repository |
|
1 | # upgrade.py - functions for in place upgrade of Mercurial repository | |
2 | # |
|
2 | # | |
3 | # Copyright (c) 2016-present, Gregory Szorc |
|
3 | # Copyright (c) 2016-present, Gregory Szorc | |
4 | # |
|
4 | # | |
5 | # This software may be used and distributed according to the terms of the |
|
5 | # This software may be used and distributed according to the terms of the | |
6 | # GNU General Public License version 2 or any later version. |
|
6 | # GNU General Public License version 2 or any later version. | |
7 |
|
7 | |||
8 | from __future__ import absolute_import |
|
8 | from __future__ import absolute_import | |
9 |
|
9 | |||
10 | import stat |
|
10 | import stat | |
11 |
|
11 | |||
12 | from ..i18n import _ |
|
12 | from ..i18n import _ | |
13 | from ..pycompat import getattr |
|
13 | from ..pycompat import getattr | |
14 | from .. import ( |
|
14 | from .. import ( | |
15 | changelog, |
|
15 | changelog, | |
16 | error, |
|
16 | error, | |
17 | filelog, |
|
17 | filelog, | |
18 | manifest, |
|
18 | manifest, | |
19 | metadata, |
|
19 | metadata, | |
20 | pycompat, |
|
20 | pycompat, | |
21 | requirements, |
|
21 | requirements, | |
22 | revlog, |
|
22 | revlog, | |
23 | scmutil, |
|
23 | scmutil, | |
24 | util, |
|
24 | util, | |
25 | vfs as vfsmod, |
|
25 | vfs as vfsmod, | |
26 | ) |
|
26 | ) | |
27 |
|
27 | |||
28 |
|
28 | |||
29 | def _revlogfrompath(repo, path): |
|
29 | def _revlogfrompath(repo, path): | |
30 | """Obtain a revlog from a repo path. |
|
30 | """Obtain a revlog from a repo path. | |
31 |
|
31 | |||
32 | An instance of the appropriate class is returned. |
|
32 | An instance of the appropriate class is returned. | |
33 | """ |
|
33 | """ | |
34 | if path == b'00changelog.i': |
|
34 | if path == b'00changelog.i': | |
35 | return changelog.changelog(repo.svfs) |
|
35 | return changelog.changelog(repo.svfs) | |
36 | elif path.endswith(b'00manifest.i'): |
|
36 | elif path.endswith(b'00manifest.i'): | |
37 | mandir = path[: -len(b'00manifest.i')] |
|
37 | mandir = path[: -len(b'00manifest.i')] | |
38 | return manifest.manifestrevlog(repo.svfs, tree=mandir) |
|
38 | return manifest.manifestrevlog(repo.svfs, tree=mandir) | |
39 | else: |
|
39 | else: | |
40 | # reverse of "/".join(("data", path + ".i")) |
|
40 | # reverse of "/".join(("data", path + ".i")) | |
41 | return filelog.filelog(repo.svfs, path[5:-2]) |
|
41 | return filelog.filelog(repo.svfs, path[5:-2]) | |
42 |
|
42 | |||
43 |
|
43 | |||
44 | def _copyrevlog(tr, destrepo, oldrl, unencodedname): |
|
44 | def _copyrevlog(tr, destrepo, oldrl, unencodedname): | |
45 | """copy all relevant files for `oldrl` into `destrepo` store |
|
45 | """copy all relevant files for `oldrl` into `destrepo` store | |
46 |
|
46 | |||
47 | Files are copied "as is" without any transformation. The copy is performed |
|
47 | Files are copied "as is" without any transformation. The copy is performed | |
48 | without extra checks. Callers are responsible for making sure the copied |
|
48 | without extra checks. Callers are responsible for making sure the copied | |
49 | content is compatible with format of the destination repository. |
|
49 | content is compatible with format of the destination repository. | |
50 | """ |
|
50 | """ | |
51 | oldrl = getattr(oldrl, '_revlog', oldrl) |
|
51 | oldrl = getattr(oldrl, '_revlog', oldrl) | |
52 | newrl = _revlogfrompath(destrepo, unencodedname) |
|
52 | newrl = _revlogfrompath(destrepo, unencodedname) | |
53 | newrl = getattr(newrl, '_revlog', newrl) |
|
53 | newrl = getattr(newrl, '_revlog', newrl) | |
54 |
|
54 | |||
55 | oldvfs = oldrl.opener |
|
55 | oldvfs = oldrl.opener | |
56 | newvfs = newrl.opener |
|
56 | newvfs = newrl.opener | |
57 | oldindex = oldvfs.join(oldrl.indexfile) |
|
57 | oldindex = oldvfs.join(oldrl.indexfile) | |
58 | newindex = newvfs.join(newrl.indexfile) |
|
58 | newindex = newvfs.join(newrl.indexfile) | |
59 | olddata = oldvfs.join(oldrl.datafile) |
|
59 | olddata = oldvfs.join(oldrl.datafile) | |
60 | newdata = newvfs.join(newrl.datafile) |
|
60 | newdata = newvfs.join(newrl.datafile) | |
61 |
|
61 | |||
62 | with newvfs(newrl.indexfile, b'w'): |
|
62 | with newvfs(newrl.indexfile, b'w'): | |
63 | pass # create all the directories |
|
63 | pass # create all the directories | |
64 |
|
64 | |||
65 | util.copyfile(oldindex, newindex) |
|
65 | util.copyfile(oldindex, newindex) | |
66 | copydata = oldrl.opener.exists(oldrl.datafile) |
|
66 | copydata = oldrl.opener.exists(oldrl.datafile) | |
67 | if copydata: |
|
67 | if copydata: | |
68 | util.copyfile(olddata, newdata) |
|
68 | util.copyfile(olddata, newdata) | |
69 |
|
69 | |||
70 | if not ( |
|
70 | if not ( | |
71 | unencodedname.endswith(b'00changelog.i') |
|
71 | unencodedname.endswith(b'00changelog.i') | |
72 | or unencodedname.endswith(b'00manifest.i') |
|
72 | or unencodedname.endswith(b'00manifest.i') | |
73 | ): |
|
73 | ): | |
74 | destrepo.svfs.fncache.add(unencodedname) |
|
74 | destrepo.svfs.fncache.add(unencodedname) | |
75 | if copydata: |
|
75 | if copydata: | |
76 | destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d') |
|
76 | destrepo.svfs.fncache.add(unencodedname[:-2] + b'.d') | |
77 |
|
77 | |||
78 |
|
78 | |||
79 | UPGRADE_CHANGELOG = b"changelog" |
|
79 | UPGRADE_CHANGELOG = b"changelog" | |
80 | UPGRADE_MANIFEST = b"manifest" |
|
80 | UPGRADE_MANIFEST = b"manifest" | |
81 | UPGRADE_FILELOGS = b"all-filelogs" |
|
81 | UPGRADE_FILELOGS = b"all-filelogs" | |
82 |
|
82 | |||
83 | UPGRADE_ALL_REVLOGS = frozenset( |
|
83 | UPGRADE_ALL_REVLOGS = frozenset( | |
84 | [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS] |
|
84 | [UPGRADE_CHANGELOG, UPGRADE_MANIFEST, UPGRADE_FILELOGS] | |
85 | ) |
|
85 | ) | |
86 |
|
86 | |||
87 |
|
87 | |||
88 | def getsidedatacompanion(srcrepo, dstrepo): |
|
88 | def getsidedatacompanion(srcrepo, dstrepo): | |
89 | sidedatacompanion = None |
|
89 | sidedatacompanion = None | |
90 | removedreqs = srcrepo.requirements - dstrepo.requirements |
|
90 | removedreqs = srcrepo.requirements - dstrepo.requirements | |
91 | addedreqs = dstrepo.requirements - srcrepo.requirements |
|
91 | addedreqs = dstrepo.requirements - srcrepo.requirements | |
92 | if requirements.SIDEDATA_REQUIREMENT in removedreqs: |
|
92 | if requirements.SIDEDATA_REQUIREMENT in removedreqs: | |
93 |
|
93 | |||
94 | def sidedatacompanion(rl, rev): |
|
94 | def sidedatacompanion(rl, rev): | |
95 | rl = getattr(rl, '_revlog', rl) |
|
95 | rl = getattr(rl, '_revlog', rl) | |
96 | if rl.flags(rev) & revlog.REVIDX_SIDEDATA: |
|
96 | if rl.flags(rev) & revlog.REVIDX_SIDEDATA: | |
97 | return True, (), {}, 0, 0 |
|
97 | return True, (), {}, 0, 0 | |
98 | return False, (), {}, 0, 0 |
|
98 | return False, (), {}, 0, 0 | |
99 |
|
99 | |||
100 | elif requirements.COPIESSDC_REQUIREMENT in addedreqs: |
|
100 | elif requirements.COPIESSDC_REQUIREMENT in addedreqs: | |
101 | sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo) |
|
101 | sidedatacompanion = metadata.getsidedataadder(srcrepo, dstrepo) | |
102 | elif requirements.COPIESSDC_REQUIREMENT in removedreqs: |
|
102 | elif requirements.COPIESSDC_REQUIREMENT in removedreqs: | |
103 | sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo) |
|
103 | sidedatacompanion = metadata.getsidedataremover(srcrepo, dstrepo) | |
104 | return sidedatacompanion |
|
104 | return sidedatacompanion | |
105 |
|
105 | |||
106 |
|
106 | |||
107 | def matchrevlog(revlogfilter, entry): |
|
107 | def matchrevlog(revlogfilter, entry): | |
108 | """check if a revlog is selected for cloning. |
|
108 | """check if a revlog is selected for cloning. | |
109 |
|
109 | |||
110 | In other words, are there any updates which need to be done on revlog |
|
110 | In other words, are there any updates which need to be done on revlog | |
111 | or it can be blindly copied. |
|
111 | or it can be blindly copied. | |
112 |
|
112 | |||
113 | The store entry is checked against the passed filter""" |
|
113 | The store entry is checked against the passed filter""" | |
114 | if entry.endswith(b'00changelog.i'): |
|
114 | if entry.endswith(b'00changelog.i'): | |
115 | return UPGRADE_CHANGELOG in revlogfilter |
|
115 | return UPGRADE_CHANGELOG in revlogfilter | |
116 | elif entry.endswith(b'00manifest.i'): |
|
116 | elif entry.endswith(b'00manifest.i'): | |
117 | return UPGRADE_MANIFEST in revlogfilter |
|
117 | return UPGRADE_MANIFEST in revlogfilter | |
118 | return UPGRADE_FILELOGS in revlogfilter |
|
118 | return UPGRADE_FILELOGS in revlogfilter | |
119 |
|
119 | |||
120 |
|
120 | |||
121 | def _perform_clone( |
|
121 | def _perform_clone( | |
122 | ui, |
|
122 | ui, | |
123 | dstrepo, |
|
123 | dstrepo, | |
124 | tr, |
|
124 | tr, | |
125 | old_revlog, |
|
125 | old_revlog, | |
126 | unencoded, |
|
126 | unencoded, | |
127 | deltareuse, |
|
127 | deltareuse, | |
128 | forcedeltabothparents, |
|
128 | forcedeltabothparents, | |
129 | revlogs, |
|
129 | revlogs, | |
130 | sidedatacompanion, |
|
130 | sidedatacompanion, | |
131 | oncopiedrevision, |
|
131 | oncopiedrevision, | |
132 | ): |
|
132 | ): | |
133 | """ returns the new revlog object created""" |
|
133 | """ returns the new revlog object created""" | |
134 | newrl = None |
|
134 | newrl = None | |
135 | if matchrevlog(revlogs, unencoded): |
|
135 | if matchrevlog(revlogs, unencoded): | |
136 | ui.note( |
|
136 | ui.note( | |
137 | _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded) |
|
137 | _(b'cloning %d revisions from %s\n') % (len(old_revlog), unencoded) | |
138 | ) |
|
138 | ) | |
139 | newrl = _revlogfrompath(dstrepo, unencoded) |
|
139 | newrl = _revlogfrompath(dstrepo, unencoded) | |
140 | old_revlog.clone( |
|
140 | old_revlog.clone( | |
141 | tr, |
|
141 | tr, | |
142 | newrl, |
|
142 | newrl, | |
143 | addrevisioncb=oncopiedrevision, |
|
143 | addrevisioncb=oncopiedrevision, | |
144 | deltareuse=deltareuse, |
|
144 | deltareuse=deltareuse, | |
145 | forcedeltabothparents=forcedeltabothparents, |
|
145 | forcedeltabothparents=forcedeltabothparents, | |
146 | sidedatacompanion=sidedatacompanion, |
|
146 | sidedatacompanion=sidedatacompanion, | |
147 | ) |
|
147 | ) | |
148 | else: |
|
148 | else: | |
149 | msg = _(b'blindly copying %s containing %i revisions\n') |
|
149 | msg = _(b'blindly copying %s containing %i revisions\n') | |
150 | ui.note(msg % (unencoded, len(old_revlog))) |
|
150 | ui.note(msg % (unencoded, len(old_revlog))) | |
151 | _copyrevlog(tr, dstrepo, old_revlog, unencoded) |
|
151 | _copyrevlog(tr, dstrepo, old_revlog, unencoded) | |
152 |
|
152 | |||
153 | newrl = _revlogfrompath(dstrepo, unencoded) |
|
153 | newrl = _revlogfrompath(dstrepo, unencoded) | |
154 | return newrl |
|
154 | return newrl | |
155 |
|
155 | |||
156 |
|
156 | |||
157 | def _clonerevlogs( |
|
157 | def _clonerevlogs( | |
158 | ui, |
|
158 | ui, | |
159 | srcrepo, |
|
159 | srcrepo, | |
160 | dstrepo, |
|
160 | dstrepo, | |
161 | tr, |
|
161 | tr, | |
162 | deltareuse, |
|
162 | deltareuse, | |
163 | forcedeltabothparents, |
|
163 | forcedeltabothparents, | |
164 | revlogs=UPGRADE_ALL_REVLOGS, |
|
164 | revlogs=UPGRADE_ALL_REVLOGS, | |
165 | ): |
|
165 | ): | |
166 | """Copy revlogs between 2 repos.""" |
|
166 | """Copy revlogs between 2 repos.""" | |
167 | revcount = 0 |
|
167 | revcount = 0 | |
168 | srcsize = 0 |
|
168 | srcsize = 0 | |
169 | srcrawsize = 0 |
|
169 | srcrawsize = 0 | |
170 | dstsize = 0 |
|
170 | dstsize = 0 | |
171 | fcount = 0 |
|
171 | fcount = 0 | |
172 | frevcount = 0 |
|
172 | frevcount = 0 | |
173 | fsrcsize = 0 |
|
173 | fsrcsize = 0 | |
174 | frawsize = 0 |
|
174 | frawsize = 0 | |
175 | fdstsize = 0 |
|
175 | fdstsize = 0 | |
176 | mcount = 0 |
|
176 | mcount = 0 | |
177 | mrevcount = 0 |
|
177 | mrevcount = 0 | |
178 | msrcsize = 0 |
|
178 | msrcsize = 0 | |
179 | mrawsize = 0 |
|
179 | mrawsize = 0 | |
180 | mdstsize = 0 |
|
180 | mdstsize = 0 | |
181 | crevcount = 0 |
|
181 | crevcount = 0 | |
182 | csrcsize = 0 |
|
182 | csrcsize = 0 | |
183 | crawsize = 0 |
|
183 | crawsize = 0 | |
184 | cdstsize = 0 |
|
184 | cdstsize = 0 | |
185 |
|
185 | |||
186 | alldatafiles = list(srcrepo.store.walk()) |
|
186 | alldatafiles = list(srcrepo.store.walk()) | |
|
187 | # mapping of data files which needs to be cloned | |||
|
188 | # key is unencoded filename | |||
|
189 | # value is revlog_object_from_srcrepo | |||
|
190 | manifests = {} | |||
|
191 | changelogs = {} | |||
|
192 | filelogs = {} | |||
187 |
|
193 | |||
188 | # Perform a pass to collect metadata. This validates we can open all |
|
194 | # Perform a pass to collect metadata. This validates we can open all | |
189 | # source files and allows a unified progress bar to be displayed. |
|
195 | # source files and allows a unified progress bar to be displayed. | |
190 | for unencoded, encoded, size in alldatafiles: |
|
196 | for unencoded, encoded, size in alldatafiles: | |
191 | if unencoded.endswith(b'.d'): |
|
197 | if unencoded.endswith(b'.d'): | |
192 | continue |
|
198 | continue | |
193 |
|
199 | |||
194 | rl = _revlogfrompath(srcrepo, unencoded) |
|
200 | rl = _revlogfrompath(srcrepo, unencoded) | |
195 |
|
201 | |||
196 | info = rl.storageinfo( |
|
202 | info = rl.storageinfo( | |
197 | exclusivefiles=True, |
|
203 | exclusivefiles=True, | |
198 | revisionscount=True, |
|
204 | revisionscount=True, | |
199 | trackedsize=True, |
|
205 | trackedsize=True, | |
200 | storedsize=True, |
|
206 | storedsize=True, | |
201 | ) |
|
207 | ) | |
202 |
|
208 | |||
203 | revcount += info[b'revisionscount'] or 0 |
|
209 | revcount += info[b'revisionscount'] or 0 | |
204 | datasize = info[b'storedsize'] or 0 |
|
210 | datasize = info[b'storedsize'] or 0 | |
205 | rawsize = info[b'trackedsize'] or 0 |
|
211 | rawsize = info[b'trackedsize'] or 0 | |
206 |
|
212 | |||
207 | srcsize += datasize |
|
213 | srcsize += datasize | |
208 | srcrawsize += rawsize |
|
214 | srcrawsize += rawsize | |
209 |
|
215 | |||
210 | # This is for the separate progress bars. |
|
216 | # This is for the separate progress bars. | |
211 | if isinstance(rl, changelog.changelog): |
|
217 | if isinstance(rl, changelog.changelog): | |
|
218 | changelogs[unencoded] = rl | |||
212 | crevcount += len(rl) |
|
219 | crevcount += len(rl) | |
213 | csrcsize += datasize |
|
220 | csrcsize += datasize | |
214 | crawsize += rawsize |
|
221 | crawsize += rawsize | |
215 | elif isinstance(rl, manifest.manifestrevlog): |
|
222 | elif isinstance(rl, manifest.manifestrevlog): | |
|
223 | manifests[unencoded] = rl | |||
216 | mcount += 1 |
|
224 | mcount += 1 | |
217 | mrevcount += len(rl) |
|
225 | mrevcount += len(rl) | |
218 | msrcsize += datasize |
|
226 | msrcsize += datasize | |
219 | mrawsize += rawsize |
|
227 | mrawsize += rawsize | |
220 | elif isinstance(rl, filelog.filelog): |
|
228 | elif isinstance(rl, filelog.filelog): | |
|
229 | filelogs[unencoded] = rl | |||
221 | fcount += 1 |
|
230 | fcount += 1 | |
222 | frevcount += len(rl) |
|
231 | frevcount += len(rl) | |
223 | fsrcsize += datasize |
|
232 | fsrcsize += datasize | |
224 | frawsize += rawsize |
|
233 | frawsize += rawsize | |
225 | else: |
|
234 | else: | |
226 | error.ProgrammingError(b'unknown revlog type') |
|
235 | error.ProgrammingError(b'unknown revlog type') | |
227 |
|
236 | |||
228 | if not revcount: |
|
237 | if not revcount: | |
229 | return |
|
238 | return | |
230 |
|
239 | |||
231 | ui.status( |
|
240 | ui.status( | |
232 | _( |
|
241 | _( | |
233 | b'migrating %d total revisions (%d in filelogs, %d in manifests, ' |
|
242 | b'migrating %d total revisions (%d in filelogs, %d in manifests, ' | |
234 | b'%d in changelog)\n' |
|
243 | b'%d in changelog)\n' | |
235 | ) |
|
244 | ) | |
236 | % (revcount, frevcount, mrevcount, crevcount) |
|
245 | % (revcount, frevcount, mrevcount, crevcount) | |
237 | ) |
|
246 | ) | |
238 | ui.status( |
|
247 | ui.status( | |
239 | _(b'migrating %s in store; %s tracked data\n') |
|
248 | _(b'migrating %s in store; %s tracked data\n') | |
240 | % ((util.bytecount(srcsize), util.bytecount(srcrawsize))) |
|
249 | % ((util.bytecount(srcsize), util.bytecount(srcrawsize))) | |
241 | ) |
|
250 | ) | |
242 |
|
251 | |||
243 | # Used to keep track of progress. |
|
252 | # Used to keep track of progress. | |
244 | progress = None |
|
253 | progress = None | |
245 |
|
254 | |||
246 | def oncopiedrevision(rl, rev, node): |
|
255 | def oncopiedrevision(rl, rev, node): | |
247 | progress.increment() |
|
256 | progress.increment() | |
248 |
|
257 | |||
249 | sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo) |
|
258 | sidedatacompanion = getsidedatacompanion(srcrepo, dstrepo) | |
250 |
|
259 | |||
251 | # Do the actual copying. |
|
260 | # Migrating filelogs | |
252 | # FUTURE this operation can be farmed off to worker processes. |
|
|||
253 | seen = set() |
|
|||
254 | for unencoded, encoded, size in alldatafiles: |
|
|||
255 | if unencoded.endswith(b'.d'): |
|
|||
256 | continue |
|
|||
257 |
|
||||
258 | oldrl = _revlogfrompath(srcrepo, unencoded) |
|
|||
259 |
|
||||
260 | if isinstance(oldrl, changelog.changelog) and b'c' not in seen: |
|
|||
261 | ui.status( |
|
|||
262 | _( |
|
|||
263 | b'finished migrating %d manifest revisions across %d ' |
|
|||
264 | b'manifests; change in size: %s\n' |
|
|||
265 | ) |
|
|||
266 | % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize)) |
|
|||
267 | ) |
|
|||
268 |
|
||||
269 | ui.status( |
|
|||
270 | _( |
|
|||
271 | b'migrating changelog containing %d revisions ' |
|
|||
272 | b'(%s in store; %s tracked data)\n' |
|
|||
273 | ) |
|
|||
274 | % ( |
|
|||
275 | crevcount, |
|
|||
276 | util.bytecount(csrcsize), |
|
|||
277 | util.bytecount(crawsize), |
|
|||
278 | ) |
|
|||
279 | ) |
|
|||
280 | seen.add(b'c') |
|
|||
281 | progress = srcrepo.ui.makeprogress( |
|
|||
282 | _(b'changelog revisions'), total=crevcount |
|
|||
283 | ) |
|
|||
284 | elif isinstance(oldrl, manifest.manifestrevlog) and b'm' not in seen: |
|
|||
285 | ui.status( |
|
|||
286 | _( |
|
|||
287 | b'finished migrating %d filelog revisions across %d ' |
|
|||
288 | b'filelogs; change in size: %s\n' |
|
|||
289 | ) |
|
|||
290 | % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize)) |
|
|||
291 | ) |
|
|||
292 |
|
||||
293 | ui.status( |
|
|||
294 | _( |
|
|||
295 | b'migrating %d manifests containing %d revisions ' |
|
|||
296 | b'(%s in store; %s tracked data)\n' |
|
|||
297 | ) |
|
|||
298 | % ( |
|
|||
299 | mcount, |
|
|||
300 | mrevcount, |
|
|||
301 | util.bytecount(msrcsize), |
|
|||
302 | util.bytecount(mrawsize), |
|
|||
303 | ) |
|
|||
304 | ) |
|
|||
305 | seen.add(b'm') |
|
|||
306 | if progress: |
|
|||
307 | progress.complete() |
|
|||
308 | progress = srcrepo.ui.makeprogress( |
|
|||
309 | _(b'manifest revisions'), total=mrevcount |
|
|||
310 | ) |
|
|||
311 | elif b'f' not in seen: |
|
|||
312 |
|
|
261 | ui.status( | |
313 |
|
|
262 | _( | |
314 |
|
|
263 | b'migrating %d filelogs containing %d revisions ' | |
315 |
|
|
264 | b'(%s in store; %s tracked data)\n' | |
316 |
|
|
265 | ) | |
317 |
|
|
266 | % ( | |
318 |
|
|
267 | fcount, | |
319 |
|
|
268 | frevcount, | |
320 |
|
|
269 | util.bytecount(fsrcsize), | |
321 |
|
|
270 | util.bytecount(frawsize), | |
322 |
|
|
271 | ) | |
323 | ) |
|
272 | ) | |
324 | seen.add(b'f') |
|
273 | progress = srcrepo.ui.makeprogress(_(b'file revisions'), total=frevcount) | |
325 | if progress: |
|
274 | for unencoded, oldrl in sorted(filelogs.items()): | |
326 | progress.complete() |
|
|||
327 | progress = srcrepo.ui.makeprogress( |
|
|||
328 | _(b'file revisions'), total=frevcount |
|
|||
329 | ) |
|
|||
330 |
|
||||
331 | newrl = _perform_clone( |
|
275 | newrl = _perform_clone( | |
332 | ui, |
|
276 | ui, | |
333 | dstrepo, |
|
277 | dstrepo, | |
334 | tr, |
|
278 | tr, | |
335 | oldrl, |
|
279 | oldrl, | |
336 | unencoded, |
|
280 | unencoded, | |
337 | deltareuse, |
|
281 | deltareuse, | |
338 | forcedeltabothparents, |
|
282 | forcedeltabothparents, | |
339 | revlogs, |
|
283 | revlogs, | |
340 | sidedatacompanion, |
|
284 | sidedatacompanion, | |
341 | oncopiedrevision, |
|
285 | oncopiedrevision, | |
342 | ) |
|
286 | ) | |
343 | info = newrl.storageinfo(storedsize=True) |
|
287 | info = newrl.storageinfo(storedsize=True) | |
344 | datasize = info[b'storedsize'] or 0 |
|
288 | datasize = info[b'storedsize'] or 0 | |
345 |
|
||||
346 | dstsize += datasize |
|
289 | dstsize += datasize | |
|
290 | fdstsize += datasize | |||
|
291 | ui.status( | |||
|
292 | _( | |||
|
293 | b'finished migrating %d filelog revisions across %d ' | |||
|
294 | b'filelogs; change in size: %s\n' | |||
|
295 | ) | |||
|
296 | % (frevcount, fcount, util.bytecount(fdstsize - fsrcsize)) | |||
|
297 | ) | |||
347 |
|
298 | |||
348 | if isinstance(newrl, changelog.changelog): |
|
299 | # Migrating manifests | |
|
300 | ui.status( | |||
|
301 | _( | |||
|
302 | b'migrating %d manifests containing %d revisions ' | |||
|
303 | b'(%s in store; %s tracked data)\n' | |||
|
304 | ) | |||
|
305 | % ( | |||
|
306 | mcount, | |||
|
307 | mrevcount, | |||
|
308 | util.bytecount(msrcsize), | |||
|
309 | util.bytecount(mrawsize), | |||
|
310 | ) | |||
|
311 | ) | |||
|
312 | if progress: | |||
|
313 | progress.complete() | |||
|
314 | progress = srcrepo.ui.makeprogress( | |||
|
315 | _(b'manifest revisions'), total=mrevcount | |||
|
316 | ) | |||
|
317 | for unencoded, oldrl in sorted(manifests.items()): | |||
|
318 | newrl = _perform_clone( | |||
|
319 | ui, | |||
|
320 | dstrepo, | |||
|
321 | tr, | |||
|
322 | oldrl, | |||
|
323 | unencoded, | |||
|
324 | deltareuse, | |||
|
325 | forcedeltabothparents, | |||
|
326 | revlogs, | |||
|
327 | sidedatacompanion, | |||
|
328 | oncopiedrevision, | |||
|
329 | ) | |||
|
330 | info = newrl.storageinfo(storedsize=True) | |||
|
331 | datasize = info[b'storedsize'] or 0 | |||
|
332 | dstsize += datasize | |||
|
333 | mdstsize += datasize | |||
|
334 | ui.status( | |||
|
335 | _( | |||
|
336 | b'finished migrating %d manifest revisions across %d ' | |||
|
337 | b'manifests; change in size: %s\n' | |||
|
338 | ) | |||
|
339 | % (mrevcount, mcount, util.bytecount(mdstsize - msrcsize)) | |||
|
340 | ) | |||
|
341 | ||||
|
342 | # Migrating changelog | |||
|
343 | ui.status( | |||
|
344 | _( | |||
|
345 | b'migrating changelog containing %d revisions ' | |||
|
346 | b'(%s in store; %s tracked data)\n' | |||
|
347 | ) | |||
|
348 | % ( | |||
|
349 | crevcount, | |||
|
350 | util.bytecount(csrcsize), | |||
|
351 | util.bytecount(crawsize), | |||
|
352 | ) | |||
|
353 | ) | |||
|
354 | if progress: | |||
|
355 | progress.complete() | |||
|
356 | progress = srcrepo.ui.makeprogress( | |||
|
357 | _(b'changelog revisions'), total=crevcount | |||
|
358 | ) | |||
|
359 | for unencoded, oldrl in sorted(changelogs.items()): | |||
|
360 | newrl = _perform_clone( | |||
|
361 | ui, | |||
|
362 | dstrepo, | |||
|
363 | tr, | |||
|
364 | oldrl, | |||
|
365 | unencoded, | |||
|
366 | deltareuse, | |||
|
367 | forcedeltabothparents, | |||
|
368 | revlogs, | |||
|
369 | sidedatacompanion, | |||
|
370 | oncopiedrevision, | |||
|
371 | ) | |||
|
372 | info = newrl.storageinfo(storedsize=True) | |||
|
373 | datasize = info[b'storedsize'] or 0 | |||
|
374 | dstsize += datasize | |||
349 |
|
|
375 | cdstsize += datasize | |
350 | elif isinstance(newrl, manifest.manifestrevlog): |
|
|||
351 | mdstsize += datasize |
|
|||
352 | else: |
|
|||
353 | fdstsize += datasize |
|
|||
354 |
|
||||
355 | progress.complete() |
|
376 | progress.complete() | |
356 |
|
||||
357 | ui.status( |
|
377 | ui.status( | |
358 | _( |
|
378 | _( | |
359 | b'finished migrating %d changelog revisions; change in size: ' |
|
379 | b'finished migrating %d changelog revisions; change in size: ' | |
360 | b'%s\n' |
|
380 | b'%s\n' | |
361 | ) |
|
381 | ) | |
362 | % (crevcount, util.bytecount(cdstsize - csrcsize)) |
|
382 | % (crevcount, util.bytecount(cdstsize - csrcsize)) | |
363 | ) |
|
383 | ) | |
364 |
|
384 | |||
365 | ui.status( |
|
385 | ui.status( | |
366 | _( |
|
386 | _( | |
367 | b'finished migrating %d total revisions; total change in store ' |
|
387 | b'finished migrating %d total revisions; total change in store ' | |
368 | b'size: %s\n' |
|
388 | b'size: %s\n' | |
369 | ) |
|
389 | ) | |
370 | % (revcount, util.bytecount(dstsize - srcsize)) |
|
390 | % (revcount, util.bytecount(dstsize - srcsize)) | |
371 | ) |
|
391 | ) | |
372 |
|
392 | |||
373 |
|
393 | |||
374 | def _filterstorefile(srcrepo, dstrepo, requirements, path, mode, st): |
|
394 | def _filterstorefile(srcrepo, dstrepo, requirements, path, mode, st): | |
375 | """Determine whether to copy a store file during upgrade. |
|
395 | """Determine whether to copy a store file during upgrade. | |
376 |
|
396 | |||
377 | This function is called when migrating store files from ``srcrepo`` to |
|
397 | This function is called when migrating store files from ``srcrepo`` to | |
378 | ``dstrepo`` as part of upgrading a repository. |
|
398 | ``dstrepo`` as part of upgrading a repository. | |
379 |
|
399 | |||
380 | Args: |
|
400 | Args: | |
381 | srcrepo: repo we are copying from |
|
401 | srcrepo: repo we are copying from | |
382 | dstrepo: repo we are copying to |
|
402 | dstrepo: repo we are copying to | |
383 | requirements: set of requirements for ``dstrepo`` |
|
403 | requirements: set of requirements for ``dstrepo`` | |
384 | path: store file being examined |
|
404 | path: store file being examined | |
385 | mode: the ``ST_MODE`` file type of ``path`` |
|
405 | mode: the ``ST_MODE`` file type of ``path`` | |
386 | st: ``stat`` data structure for ``path`` |
|
406 | st: ``stat`` data structure for ``path`` | |
387 |
|
407 | |||
388 | Function should return ``True`` if the file is to be copied. |
|
408 | Function should return ``True`` if the file is to be copied. | |
389 | """ |
|
409 | """ | |
390 | # Skip revlogs. |
|
410 | # Skip revlogs. | |
391 | if path.endswith((b'.i', b'.d', b'.n', b'.nd')): |
|
411 | if path.endswith((b'.i', b'.d', b'.n', b'.nd')): | |
392 | return False |
|
412 | return False | |
393 | # Skip transaction related files. |
|
413 | # Skip transaction related files. | |
394 | if path.startswith(b'undo'): |
|
414 | if path.startswith(b'undo'): | |
395 | return False |
|
415 | return False | |
396 | # Only copy regular files. |
|
416 | # Only copy regular files. | |
397 | if mode != stat.S_IFREG: |
|
417 | if mode != stat.S_IFREG: | |
398 | return False |
|
418 | return False | |
399 | # Skip other skipped files. |
|
419 | # Skip other skipped files. | |
400 | if path in (b'lock', b'fncache'): |
|
420 | if path in (b'lock', b'fncache'): | |
401 | return False |
|
421 | return False | |
402 |
|
422 | |||
403 | return True |
|
423 | return True | |
404 |
|
424 | |||
405 |
|
425 | |||
406 | def _finishdatamigration(ui, srcrepo, dstrepo, requirements): |
|
426 | def _finishdatamigration(ui, srcrepo, dstrepo, requirements): | |
407 | """Hook point for extensions to perform additional actions during upgrade. |
|
427 | """Hook point for extensions to perform additional actions during upgrade. | |
408 |
|
428 | |||
409 | This function is called after revlogs and store files have been copied but |
|
429 | This function is called after revlogs and store files have been copied but | |
410 | before the new store is swapped into the original location. |
|
430 | before the new store is swapped into the original location. | |
411 | """ |
|
431 | """ | |
412 |
|
432 | |||
413 |
|
433 | |||
414 | def upgrade(ui, srcrepo, dstrepo, upgrade_op): |
|
434 | def upgrade(ui, srcrepo, dstrepo, upgrade_op): | |
415 | """Do the low-level work of upgrading a repository. |
|
435 | """Do the low-level work of upgrading a repository. | |
416 |
|
436 | |||
417 | The upgrade is effectively performed as a copy between a source |
|
437 | The upgrade is effectively performed as a copy between a source | |
418 | repository and a temporary destination repository. |
|
438 | repository and a temporary destination repository. | |
419 |
|
439 | |||
420 | The source repository is unmodified for as long as possible so the |
|
440 | The source repository is unmodified for as long as possible so the | |
421 | upgrade can abort at any time without causing loss of service for |
|
441 | upgrade can abort at any time without causing loss of service for | |
422 | readers and without corrupting the source repository. |
|
442 | readers and without corrupting the source repository. | |
423 | """ |
|
443 | """ | |
424 | assert srcrepo.currentwlock() |
|
444 | assert srcrepo.currentwlock() | |
425 | assert dstrepo.currentwlock() |
|
445 | assert dstrepo.currentwlock() | |
426 |
|
446 | |||
427 | ui.status( |
|
447 | ui.status( | |
428 | _( |
|
448 | _( | |
429 | b'(it is safe to interrupt this process any time before ' |
|
449 | b'(it is safe to interrupt this process any time before ' | |
430 | b'data migration completes)\n' |
|
450 | b'data migration completes)\n' | |
431 | ) |
|
451 | ) | |
432 | ) |
|
452 | ) | |
433 |
|
453 | |||
434 | if upgrade_op.has_action(b're-delta-all'): |
|
454 | if upgrade_op.has_action(b're-delta-all'): | |
435 | deltareuse = revlog.revlog.DELTAREUSENEVER |
|
455 | deltareuse = revlog.revlog.DELTAREUSENEVER | |
436 | elif upgrade_op.has_action(b're-delta-parent'): |
|
456 | elif upgrade_op.has_action(b're-delta-parent'): | |
437 | deltareuse = revlog.revlog.DELTAREUSESAMEREVS |
|
457 | deltareuse = revlog.revlog.DELTAREUSESAMEREVS | |
438 | elif upgrade_op.has_action(b're-delta-multibase'): |
|
458 | elif upgrade_op.has_action(b're-delta-multibase'): | |
439 | deltareuse = revlog.revlog.DELTAREUSESAMEREVS |
|
459 | deltareuse = revlog.revlog.DELTAREUSESAMEREVS | |
440 | elif upgrade_op.has_action(b're-delta-fulladd'): |
|
460 | elif upgrade_op.has_action(b're-delta-fulladd'): | |
441 | deltareuse = revlog.revlog.DELTAREUSEFULLADD |
|
461 | deltareuse = revlog.revlog.DELTAREUSEFULLADD | |
442 | else: |
|
462 | else: | |
443 | deltareuse = revlog.revlog.DELTAREUSEALWAYS |
|
463 | deltareuse = revlog.revlog.DELTAREUSEALWAYS | |
444 |
|
464 | |||
445 | with dstrepo.transaction(b'upgrade') as tr: |
|
465 | with dstrepo.transaction(b'upgrade') as tr: | |
446 | _clonerevlogs( |
|
466 | _clonerevlogs( | |
447 | ui, |
|
467 | ui, | |
448 | srcrepo, |
|
468 | srcrepo, | |
449 | dstrepo, |
|
469 | dstrepo, | |
450 | tr, |
|
470 | tr, | |
451 | deltareuse, |
|
471 | deltareuse, | |
452 | upgrade_op.has_action(b're-delta-multibase'), |
|
472 | upgrade_op.has_action(b're-delta-multibase'), | |
453 | revlogs=upgrade_op.revlogs_to_process, |
|
473 | revlogs=upgrade_op.revlogs_to_process, | |
454 | ) |
|
474 | ) | |
455 |
|
475 | |||
456 | # Now copy other files in the store directory. |
|
476 | # Now copy other files in the store directory. | |
457 | # The sorted() makes execution deterministic. |
|
477 | # The sorted() makes execution deterministic. | |
458 | for p, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)): |
|
478 | for p, kind, st in sorted(srcrepo.store.vfs.readdir(b'', stat=True)): | |
459 | if not _filterstorefile( |
|
479 | if not _filterstorefile( | |
460 | srcrepo, dstrepo, upgrade_op.new_requirements, p, kind, st |
|
480 | srcrepo, dstrepo, upgrade_op.new_requirements, p, kind, st | |
461 | ): |
|
481 | ): | |
462 | continue |
|
482 | continue | |
463 |
|
483 | |||
464 | srcrepo.ui.status(_(b'copying %s\n') % p) |
|
484 | srcrepo.ui.status(_(b'copying %s\n') % p) | |
465 | src = srcrepo.store.rawvfs.join(p) |
|
485 | src = srcrepo.store.rawvfs.join(p) | |
466 | dst = dstrepo.store.rawvfs.join(p) |
|
486 | dst = dstrepo.store.rawvfs.join(p) | |
467 | util.copyfile(src, dst, copystat=True) |
|
487 | util.copyfile(src, dst, copystat=True) | |
468 |
|
488 | |||
469 | _finishdatamigration(ui, srcrepo, dstrepo, requirements) |
|
489 | _finishdatamigration(ui, srcrepo, dstrepo, requirements) | |
470 |
|
490 | |||
471 | ui.status(_(b'data fully migrated to temporary repository\n')) |
|
491 | ui.status(_(b'data fully migrated to temporary repository\n')) | |
472 |
|
492 | |||
473 | backuppath = pycompat.mkdtemp(prefix=b'upgradebackup.', dir=srcrepo.path) |
|
493 | backuppath = pycompat.mkdtemp(prefix=b'upgradebackup.', dir=srcrepo.path) | |
474 | backupvfs = vfsmod.vfs(backuppath) |
|
494 | backupvfs = vfsmod.vfs(backuppath) | |
475 |
|
495 | |||
476 | # Make a backup of requires file first, as it is the first to be modified. |
|
496 | # Make a backup of requires file first, as it is the first to be modified. | |
477 | util.copyfile(srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires')) |
|
497 | util.copyfile(srcrepo.vfs.join(b'requires'), backupvfs.join(b'requires')) | |
478 |
|
498 | |||
479 | # We install an arbitrary requirement that clients must not support |
|
499 | # We install an arbitrary requirement that clients must not support | |
480 | # as a mechanism to lock out new clients during the data swap. This is |
|
500 | # as a mechanism to lock out new clients during the data swap. This is | |
481 | # better than allowing a client to continue while the repository is in |
|
501 | # better than allowing a client to continue while the repository is in | |
482 | # an inconsistent state. |
|
502 | # an inconsistent state. | |
483 | ui.status( |
|
503 | ui.status( | |
484 | _( |
|
504 | _( | |
485 | b'marking source repository as being upgraded; clients will be ' |
|
505 | b'marking source repository as being upgraded; clients will be ' | |
486 | b'unable to read from repository\n' |
|
506 | b'unable to read from repository\n' | |
487 | ) |
|
507 | ) | |
488 | ) |
|
508 | ) | |
489 | scmutil.writereporequirements( |
|
509 | scmutil.writereporequirements( | |
490 | srcrepo, srcrepo.requirements | {b'upgradeinprogress'} |
|
510 | srcrepo, srcrepo.requirements | {b'upgradeinprogress'} | |
491 | ) |
|
511 | ) | |
492 |
|
512 | |||
493 | ui.status(_(b'starting in-place swap of repository data\n')) |
|
513 | ui.status(_(b'starting in-place swap of repository data\n')) | |
494 | ui.status(_(b'replaced files will be backed up at %s\n') % backuppath) |
|
514 | ui.status(_(b'replaced files will be backed up at %s\n') % backuppath) | |
495 |
|
515 | |||
496 | # Now swap in the new store directory. Doing it as a rename should make |
|
516 | # Now swap in the new store directory. Doing it as a rename should make | |
497 | # the operation nearly instantaneous and atomic (at least in well-behaved |
|
517 | # the operation nearly instantaneous and atomic (at least in well-behaved | |
498 | # environments). |
|
518 | # environments). | |
499 | ui.status(_(b'replacing store...\n')) |
|
519 | ui.status(_(b'replacing store...\n')) | |
500 | tstart = util.timer() |
|
520 | tstart = util.timer() | |
501 | util.rename(srcrepo.spath, backupvfs.join(b'store')) |
|
521 | util.rename(srcrepo.spath, backupvfs.join(b'store')) | |
502 | util.rename(dstrepo.spath, srcrepo.spath) |
|
522 | util.rename(dstrepo.spath, srcrepo.spath) | |
503 | elapsed = util.timer() - tstart |
|
523 | elapsed = util.timer() - tstart | |
504 | ui.status( |
|
524 | ui.status( | |
505 | _( |
|
525 | _( | |
506 | b'store replacement complete; repository was inconsistent for ' |
|
526 | b'store replacement complete; repository was inconsistent for ' | |
507 | b'%0.1fs\n' |
|
527 | b'%0.1fs\n' | |
508 | ) |
|
528 | ) | |
509 | % elapsed |
|
529 | % elapsed | |
510 | ) |
|
530 | ) | |
511 |
|
531 | |||
512 | # We first write the requirements file. Any new requirements will lock |
|
532 | # We first write the requirements file. Any new requirements will lock | |
513 | # out legacy clients. |
|
533 | # out legacy clients. | |
514 | ui.status( |
|
534 | ui.status( | |
515 | _( |
|
535 | _( | |
516 | b'finalizing requirements file and making repository readable ' |
|
536 | b'finalizing requirements file and making repository readable ' | |
517 | b'again\n' |
|
537 | b'again\n' | |
518 | ) |
|
538 | ) | |
519 | ) |
|
539 | ) | |
520 | scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements) |
|
540 | scmutil.writereporequirements(srcrepo, upgrade_op.new_requirements) | |
521 |
|
541 | |||
522 | # The lock file from the old store won't be removed because nothing has a |
|
542 | # The lock file from the old store won't be removed because nothing has a | |
523 | # reference to its new location. So clean it up manually. Alternatively, we |
|
543 | # reference to its new location. So clean it up manually. Alternatively, we | |
524 | # could update srcrepo.svfs and other variables to point to the new |
|
544 | # could update srcrepo.svfs and other variables to point to the new | |
525 | # location. This is simpler. |
|
545 | # location. This is simpler. | |
526 | backupvfs.unlink(b'store/lock') |
|
546 | backupvfs.unlink(b'store/lock') | |
527 |
|
547 | |||
528 | return backuppath |
|
548 | return backuppath |
General Comments 0
You need to be logged in to leave comments.
Login now