##// END OF EJS Templates
lfs: use 'ui' provided to `upgrade` for output, instead of stealing srcrepo's...
Matt Harbison -
r35398:a7ff4071 default
parent child Browse files
Show More
@@ -1,323 +1,323 b''
1 # wrapper.py - methods wrapping core mercurial logic
1 # wrapper.py - methods wrapping core mercurial logic
2 #
2 #
3 # Copyright 2017 Facebook, Inc.
3 # Copyright 2017 Facebook, Inc.
4 #
4 #
5 # This software may be used and distributed according to the terms of the
5 # This software may be used and distributed according to the terms of the
6 # GNU General Public License version 2 or any later version.
6 # GNU General Public License version 2 or any later version.
7
7
8 from __future__ import absolute_import
8 from __future__ import absolute_import
9
9
10 import hashlib
10 import hashlib
11
11
12 from mercurial.i18n import _
12 from mercurial.i18n import _
13 from mercurial.node import bin, nullid, short
13 from mercurial.node import bin, nullid, short
14
14
15 from mercurial import (
15 from mercurial import (
16 error,
16 error,
17 filelog,
17 filelog,
18 revlog,
18 revlog,
19 util,
19 util,
20 )
20 )
21
21
22 from ..largefiles import lfutil
22 from ..largefiles import lfutil
23
23
24 from . import (
24 from . import (
25 blobstore,
25 blobstore,
26 pointer,
26 pointer,
27 )
27 )
28
28
29 def supportedoutgoingversions(orig, repo):
29 def supportedoutgoingversions(orig, repo):
30 versions = orig(repo)
30 versions = orig(repo)
31 versions.discard('01')
31 versions.discard('01')
32 versions.discard('02')
32 versions.discard('02')
33 versions.add('03')
33 versions.add('03')
34 return versions
34 return versions
35
35
36 def allsupportedversions(orig, ui):
36 def allsupportedversions(orig, ui):
37 versions = orig(ui)
37 versions = orig(ui)
38 versions.add('03')
38 versions.add('03')
39 return versions
39 return versions
40
40
41 def bypasscheckhash(self, text):
41 def bypasscheckhash(self, text):
42 return False
42 return False
43
43
44 def readfromstore(self, text):
44 def readfromstore(self, text):
45 """Read filelog content from local blobstore transform for flagprocessor.
45 """Read filelog content from local blobstore transform for flagprocessor.
46
46
47 Default tranform for flagprocessor, returning contents from blobstore.
47 Default tranform for flagprocessor, returning contents from blobstore.
48 Returns a 2-typle (text, validatehash) where validatehash is True as the
48 Returns a 2-typle (text, validatehash) where validatehash is True as the
49 contents of the blobstore should be checked using checkhash.
49 contents of the blobstore should be checked using checkhash.
50 """
50 """
51 p = pointer.deserialize(text)
51 p = pointer.deserialize(text)
52 oid = p.oid()
52 oid = p.oid()
53 store = self.opener.lfslocalblobstore
53 store = self.opener.lfslocalblobstore
54 if not store.has(oid):
54 if not store.has(oid):
55 p.filename = getattr(self, 'indexfile', None)
55 p.filename = getattr(self, 'indexfile', None)
56 self.opener.lfsremoteblobstore.readbatch([p], store)
56 self.opener.lfsremoteblobstore.readbatch([p], store)
57 text = store.read(oid)
57 text = store.read(oid)
58
58
59 # pack hg filelog metadata
59 # pack hg filelog metadata
60 hgmeta = {}
60 hgmeta = {}
61 for k in p.keys():
61 for k in p.keys():
62 if k.startswith('x-hg-'):
62 if k.startswith('x-hg-'):
63 name = k[len('x-hg-'):]
63 name = k[len('x-hg-'):]
64 hgmeta[name] = p[k]
64 hgmeta[name] = p[k]
65 if hgmeta or text.startswith('\1\n'):
65 if hgmeta or text.startswith('\1\n'):
66 text = filelog.packmeta(hgmeta, text)
66 text = filelog.packmeta(hgmeta, text)
67
67
68 return (text, True)
68 return (text, True)
69
69
70 def writetostore(self, text):
70 def writetostore(self, text):
71 # hg filelog metadata (includes rename, etc)
71 # hg filelog metadata (includes rename, etc)
72 hgmeta, offset = filelog.parsemeta(text)
72 hgmeta, offset = filelog.parsemeta(text)
73 if offset and offset > 0:
73 if offset and offset > 0:
74 # lfs blob does not contain hg filelog metadata
74 # lfs blob does not contain hg filelog metadata
75 text = text[offset:]
75 text = text[offset:]
76
76
77 # git-lfs only supports sha256
77 # git-lfs only supports sha256
78 oid = hashlib.sha256(text).hexdigest()
78 oid = hashlib.sha256(text).hexdigest()
79 self.opener.lfslocalblobstore.write(oid, text)
79 self.opener.lfslocalblobstore.write(oid, text)
80
80
81 # replace contents with metadata
81 # replace contents with metadata
82 longoid = 'sha256:%s' % oid
82 longoid = 'sha256:%s' % oid
83 metadata = pointer.gitlfspointer(oid=longoid, size=str(len(text)))
83 metadata = pointer.gitlfspointer(oid=longoid, size=str(len(text)))
84
84
85 # by default, we expect the content to be binary. however, LFS could also
85 # by default, we expect the content to be binary. however, LFS could also
86 # be used for non-binary content. add a special entry for non-binary data.
86 # be used for non-binary content. add a special entry for non-binary data.
87 # this will be used by filectx.isbinary().
87 # this will be used by filectx.isbinary().
88 if not util.binary(text):
88 if not util.binary(text):
89 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
89 # not hg filelog metadata (affecting commit hash), no "x-hg-" prefix
90 metadata['x-is-binary'] = '0'
90 metadata['x-is-binary'] = '0'
91
91
92 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
92 # translate hg filelog metadata to lfs metadata with "x-hg-" prefix
93 if hgmeta is not None:
93 if hgmeta is not None:
94 for k, v in hgmeta.iteritems():
94 for k, v in hgmeta.iteritems():
95 metadata['x-hg-%s' % k] = v
95 metadata['x-hg-%s' % k] = v
96
96
97 rawtext = metadata.serialize()
97 rawtext = metadata.serialize()
98 return (rawtext, False)
98 return (rawtext, False)
99
99
100 def _islfs(rlog, node=None, rev=None):
100 def _islfs(rlog, node=None, rev=None):
101 if rev is None:
101 if rev is None:
102 if node is None:
102 if node is None:
103 # both None - likely working copy content where node is not ready
103 # both None - likely working copy content where node is not ready
104 return False
104 return False
105 rev = rlog.rev(node)
105 rev = rlog.rev(node)
106 else:
106 else:
107 node = rlog.node(rev)
107 node = rlog.node(rev)
108 if node == nullid:
108 if node == nullid:
109 return False
109 return False
110 flags = rlog.flags(rev)
110 flags = rlog.flags(rev)
111 return bool(flags & revlog.REVIDX_EXTSTORED)
111 return bool(flags & revlog.REVIDX_EXTSTORED)
112
112
113 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
113 def filelogaddrevision(orig, self, text, transaction, link, p1, p2,
114 cachedelta=None, node=None,
114 cachedelta=None, node=None,
115 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
115 flags=revlog.REVIDX_DEFAULT_FLAGS, **kwds):
116 threshold = self.opener.options['lfsthreshold']
116 threshold = self.opener.options['lfsthreshold']
117 textlen = len(text)
117 textlen = len(text)
118 # exclude hg rename meta from file size
118 # exclude hg rename meta from file size
119 meta, offset = filelog.parsemeta(text)
119 meta, offset = filelog.parsemeta(text)
120 if offset:
120 if offset:
121 textlen -= offset
121 textlen -= offset
122
122
123 if threshold and textlen > threshold:
123 if threshold and textlen > threshold:
124 flags |= revlog.REVIDX_EXTSTORED
124 flags |= revlog.REVIDX_EXTSTORED
125
125
126 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
126 return orig(self, text, transaction, link, p1, p2, cachedelta=cachedelta,
127 node=node, flags=flags, **kwds)
127 node=node, flags=flags, **kwds)
128
128
129 def filelogrenamed(orig, self, node):
129 def filelogrenamed(orig, self, node):
130 if _islfs(self, node):
130 if _islfs(self, node):
131 rawtext = self.revision(node, raw=True)
131 rawtext = self.revision(node, raw=True)
132 if not rawtext:
132 if not rawtext:
133 return False
133 return False
134 metadata = pointer.deserialize(rawtext)
134 metadata = pointer.deserialize(rawtext)
135 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
135 if 'x-hg-copy' in metadata and 'x-hg-copyrev' in metadata:
136 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
136 return metadata['x-hg-copy'], bin(metadata['x-hg-copyrev'])
137 else:
137 else:
138 return False
138 return False
139 return orig(self, node)
139 return orig(self, node)
140
140
141 def filelogsize(orig, self, rev):
141 def filelogsize(orig, self, rev):
142 if _islfs(self, rev=rev):
142 if _islfs(self, rev=rev):
143 # fast path: use lfs metadata to answer size
143 # fast path: use lfs metadata to answer size
144 rawtext = self.revision(rev, raw=True)
144 rawtext = self.revision(rev, raw=True)
145 metadata = pointer.deserialize(rawtext)
145 metadata = pointer.deserialize(rawtext)
146 return int(metadata['size'])
146 return int(metadata['size'])
147 return orig(self, rev)
147 return orig(self, rev)
148
148
149 def filectxcmp(orig, self, fctx):
149 def filectxcmp(orig, self, fctx):
150 """returns True if text is different than fctx"""
150 """returns True if text is different than fctx"""
151 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
151 # some fctx (ex. hg-git) is not based on basefilectx and do not have islfs
152 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
152 if self.islfs() and getattr(fctx, 'islfs', lambda: False)():
153 # fast path: check LFS oid
153 # fast path: check LFS oid
154 p1 = pointer.deserialize(self.rawdata())
154 p1 = pointer.deserialize(self.rawdata())
155 p2 = pointer.deserialize(fctx.rawdata())
155 p2 = pointer.deserialize(fctx.rawdata())
156 return p1.oid() != p2.oid()
156 return p1.oid() != p2.oid()
157 return orig(self, fctx)
157 return orig(self, fctx)
158
158
159 def filectxisbinary(orig, self):
159 def filectxisbinary(orig, self):
160 if self.islfs():
160 if self.islfs():
161 # fast path: use lfs metadata to answer isbinary
161 # fast path: use lfs metadata to answer isbinary
162 metadata = pointer.deserialize(self.rawdata())
162 metadata = pointer.deserialize(self.rawdata())
163 # if lfs metadata says nothing, assume it's binary by default
163 # if lfs metadata says nothing, assume it's binary by default
164 return bool(int(metadata.get('x-is-binary', 1)))
164 return bool(int(metadata.get('x-is-binary', 1)))
165 return orig(self)
165 return orig(self)
166
166
167 def filectxislfs(self):
167 def filectxislfs(self):
168 return _islfs(self.filelog(), self.filenode())
168 return _islfs(self.filelog(), self.filenode())
169
169
170 def convertsink(orig, sink):
170 def convertsink(orig, sink):
171 sink = orig(sink)
171 sink = orig(sink)
172 if sink.repotype == 'hg':
172 if sink.repotype == 'hg':
173 class lfssink(sink.__class__):
173 class lfssink(sink.__class__):
174 def putcommit(self, files, copies, parents, commit, source, revmap,
174 def putcommit(self, files, copies, parents, commit, source, revmap,
175 full, cleanp2):
175 full, cleanp2):
176 pc = super(lfssink, self).putcommit
176 pc = super(lfssink, self).putcommit
177 node = pc(files, copies, parents, commit, source, revmap, full,
177 node = pc(files, copies, parents, commit, source, revmap, full,
178 cleanp2)
178 cleanp2)
179
179
180 if 'lfs' not in self.repo.requirements:
180 if 'lfs' not in self.repo.requirements:
181 ctx = self.repo[node]
181 ctx = self.repo[node]
182
182
183 # The file list may contain removed files, so check for
183 # The file list may contain removed files, so check for
184 # membership before assuming it is in the context.
184 # membership before assuming it is in the context.
185 if any(f in ctx and ctx[f].islfs() for f, n in files):
185 if any(f in ctx and ctx[f].islfs() for f, n in files):
186 self.repo.requirements.add('lfs')
186 self.repo.requirements.add('lfs')
187 self.repo._writerequirements()
187 self.repo._writerequirements()
188
188
189 # Permanently enable lfs locally
189 # Permanently enable lfs locally
190 with self.repo.vfs('hgrc', 'a', text=True) as fp:
190 with self.repo.vfs('hgrc', 'a', text=True) as fp:
191 fp.write('\n[extensions]\nlfs=\n')
191 fp.write('\n[extensions]\nlfs=\n')
192
192
193 return node
193 return node
194
194
195 sink.__class__ = lfssink
195 sink.__class__ = lfssink
196
196
197 return sink
197 return sink
198
198
199 def vfsinit(orig, self, othervfs):
199 def vfsinit(orig, self, othervfs):
200 orig(self, othervfs)
200 orig(self, othervfs)
201 # copy lfs related options
201 # copy lfs related options
202 for k, v in othervfs.options.items():
202 for k, v in othervfs.options.items():
203 if k.startswith('lfs'):
203 if k.startswith('lfs'):
204 self.options[k] = v
204 self.options[k] = v
205 # also copy lfs blobstores. note: this can run before reposetup, so lfs
205 # also copy lfs blobstores. note: this can run before reposetup, so lfs
206 # blobstore attributes are not always ready at this time.
206 # blobstore attributes are not always ready at this time.
207 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
207 for name in ['lfslocalblobstore', 'lfsremoteblobstore']:
208 if util.safehasattr(othervfs, name):
208 if util.safehasattr(othervfs, name):
209 setattr(self, name, getattr(othervfs, name))
209 setattr(self, name, getattr(othervfs, name))
210
210
211 def hgclone(orig, ui, opts, *args, **kwargs):
211 def hgclone(orig, ui, opts, *args, **kwargs):
212 result = orig(ui, opts, *args, **kwargs)
212 result = orig(ui, opts, *args, **kwargs)
213
213
214 if result is not None:
214 if result is not None:
215 sourcerepo, destrepo = result
215 sourcerepo, destrepo = result
216 repo = destrepo.local()
216 repo = destrepo.local()
217
217
218 # When cloning to a remote repo (like through SSH), no repo is available
218 # When cloning to a remote repo (like through SSH), no repo is available
219 # from the peer. Therefore the hgrc can't be updated.
219 # from the peer. Therefore the hgrc can't be updated.
220 if not repo:
220 if not repo:
221 return result
221 return result
222
222
223 # If lfs is required for this repo, permanently enable it locally
223 # If lfs is required for this repo, permanently enable it locally
224 if 'lfs' in repo.requirements:
224 if 'lfs' in repo.requirements:
225 with repo.vfs('hgrc', 'a', text=True) as fp:
225 with repo.vfs('hgrc', 'a', text=True) as fp:
226 fp.write('\n[extensions]\nlfs=\n')
226 fp.write('\n[extensions]\nlfs=\n')
227
227
228 return result
228 return result
229
229
230 def hgpostshare(orig, sourcerepo, destrepo, bookmarks=True, defaultpath=None):
230 def hgpostshare(orig, sourcerepo, destrepo, bookmarks=True, defaultpath=None):
231 orig(sourcerepo, destrepo, bookmarks, defaultpath)
231 orig(sourcerepo, destrepo, bookmarks, defaultpath)
232
232
233 # If lfs is required for this repo, permanently enable it locally
233 # If lfs is required for this repo, permanently enable it locally
234 if 'lfs' in destrepo.requirements:
234 if 'lfs' in destrepo.requirements:
235 with destrepo.vfs('hgrc', 'a', text=True) as fp:
235 with destrepo.vfs('hgrc', 'a', text=True) as fp:
236 fp.write('\n[extensions]\nlfs=\n')
236 fp.write('\n[extensions]\nlfs=\n')
237
237
238 def _canskipupload(repo):
238 def _canskipupload(repo):
239 # if remotestore is a null store, upload is a no-op and can be skipped
239 # if remotestore is a null store, upload is a no-op and can be skipped
240 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
240 return isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
241
241
242 def candownload(repo):
242 def candownload(repo):
243 # if remotestore is a null store, downloads will lead to nothing
243 # if remotestore is a null store, downloads will lead to nothing
244 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
244 return not isinstance(repo.svfs.lfsremoteblobstore, blobstore._nullremote)
245
245
246 def uploadblobsfromrevs(repo, revs):
246 def uploadblobsfromrevs(repo, revs):
247 '''upload lfs blobs introduced by revs
247 '''upload lfs blobs introduced by revs
248
248
249 Note: also used by other extensions e. g. infinitepush. avoid renaming.
249 Note: also used by other extensions e. g. infinitepush. avoid renaming.
250 '''
250 '''
251 if _canskipupload(repo):
251 if _canskipupload(repo):
252 return
252 return
253 pointers = extractpointers(repo, revs)
253 pointers = extractpointers(repo, revs)
254 uploadblobs(repo, pointers)
254 uploadblobs(repo, pointers)
255
255
256 def prepush(pushop):
256 def prepush(pushop):
257 """Prepush hook.
257 """Prepush hook.
258
258
259 Read through the revisions to push, looking for filelog entries that can be
259 Read through the revisions to push, looking for filelog entries that can be
260 deserialized into metadata so that we can block the push on their upload to
260 deserialized into metadata so that we can block the push on their upload to
261 the remote blobstore.
261 the remote blobstore.
262 """
262 """
263 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
263 return uploadblobsfromrevs(pushop.repo, pushop.outgoing.missing)
264
264
265 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
265 def writenewbundle(orig, ui, repo, source, filename, bundletype, outgoing,
266 *args, **kwargs):
266 *args, **kwargs):
267 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
267 """upload LFS blobs added by outgoing revisions on 'hg bundle'"""
268 uploadblobsfromrevs(repo, outgoing.missing)
268 uploadblobsfromrevs(repo, outgoing.missing)
269 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
269 return orig(ui, repo, source, filename, bundletype, outgoing, *args,
270 **kwargs)
270 **kwargs)
271
271
272 def extractpointers(repo, revs):
272 def extractpointers(repo, revs):
273 """return a list of lfs pointers added by given revs"""
273 """return a list of lfs pointers added by given revs"""
274 ui = repo.ui
274 ui = repo.ui
275 if ui.debugflag:
275 if ui.debugflag:
276 ui.write(_('lfs: computing set of blobs to upload\n'))
276 ui.write(_('lfs: computing set of blobs to upload\n'))
277 pointers = {}
277 pointers = {}
278 for r in revs:
278 for r in revs:
279 ctx = repo[r]
279 ctx = repo[r]
280 for p in pointersfromctx(ctx).values():
280 for p in pointersfromctx(ctx).values():
281 pointers[p.oid()] = p
281 pointers[p.oid()] = p
282 return pointers.values()
282 return pointers.values()
283
283
284 def pointersfromctx(ctx):
284 def pointersfromctx(ctx):
285 """return a dict {path: pointer} for given single changectx"""
285 """return a dict {path: pointer} for given single changectx"""
286 result = {}
286 result = {}
287 for f in ctx.files():
287 for f in ctx.files():
288 if f not in ctx:
288 if f not in ctx:
289 continue
289 continue
290 fctx = ctx[f]
290 fctx = ctx[f]
291 if not _islfs(fctx.filelog(), fctx.filenode()):
291 if not _islfs(fctx.filelog(), fctx.filenode()):
292 continue
292 continue
293 try:
293 try:
294 result[f] = pointer.deserialize(fctx.rawdata())
294 result[f] = pointer.deserialize(fctx.rawdata())
295 except pointer.InvalidPointer as ex:
295 except pointer.InvalidPointer as ex:
296 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
296 raise error.Abort(_('lfs: corrupted pointer (%s@%s): %s\n')
297 % (f, short(ctx.node()), ex))
297 % (f, short(ctx.node()), ex))
298 return result
298 return result
299
299
300 def uploadblobs(repo, pointers):
300 def uploadblobs(repo, pointers):
301 """upload given pointers from local blobstore"""
301 """upload given pointers from local blobstore"""
302 if not pointers:
302 if not pointers:
303 return
303 return
304
304
305 remoteblob = repo.svfs.lfsremoteblobstore
305 remoteblob = repo.svfs.lfsremoteblobstore
306 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
306 remoteblob.writebatch(pointers, repo.svfs.lfslocalblobstore)
307
307
308 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
308 def upgradefinishdatamigration(orig, ui, srcrepo, dstrepo, requirements):
309 orig(ui, srcrepo, dstrepo, requirements)
309 orig(ui, srcrepo, dstrepo, requirements)
310
310
311 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
311 srclfsvfs = srcrepo.svfs.lfslocalblobstore.vfs
312 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
312 dstlfsvfs = dstrepo.svfs.lfslocalblobstore.vfs
313
313
314 for dirpath, dirs, files in srclfsvfs.walk():
314 for dirpath, dirs, files in srclfsvfs.walk():
315 for oid in files:
315 for oid in files:
316 srcrepo.ui.write(_('copying lfs blob %s\n') % oid)
316 ui.write(_('copying lfs blob %s\n') % oid)
317 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
317 lfutil.link(srclfsvfs.join(oid), dstlfsvfs.join(oid))
318
318
319 def upgraderequirements(orig, repo):
319 def upgraderequirements(orig, repo):
320 reqs = orig(repo)
320 reqs = orig(repo)
321 if 'lfs' in repo.requirements:
321 if 'lfs' in repo.requirements:
322 reqs.add('lfs')
322 reqs.add('lfs')
323 return reqs
323 return reqs
General Comments 0
You need to be logged in to leave comments. Login now