##// END OF EJS Templates
lfs: check content length after downloading content...
Matt Harbison -
r44929:0ee0a3f6 default
parent child Browse files
Show More
@@ -155,15 +155,29 b' class local(object):'
155 155
156 156 return self.vfs(oid, b'rb')
157 157
158 def download(self, oid, src):
158 def download(self, oid, src, content_length):
159 159 """Read the blob from the remote source in chunks, verify the content,
160 160 and write to this local blobstore."""
161 161 sha256 = hashlib.sha256()
162 size = 0
162 163
163 164 with self.vfs(oid, b'wb', atomictemp=True) as fp:
164 165 for chunk in util.filechunkiter(src, size=1048576):
165 166 fp.write(chunk)
166 167 sha256.update(chunk)
168 size += len(chunk)
169
170 # If the server advertised a length longer than what we actually
171 # received, then we should expect that the server crashed while
172 # producing the response (but the server has no way of telling us
173 # that), and we really don't need to try to write the response to
174 # the localstore, because it's not going to match the expected.
175 if content_length is not None and int(content_length) != size:
176 msg = (
177 b"Response length (%s) does not match Content-Length "
178 b"header (%d): likely server-side crash"
179 )
180 raise LfsRemoteError(_(msg) % (size, int(content_length)))
167 181
168 182 realoid = node.hex(sha256.digest())
169 183 if realoid != oid:
@@ -492,6 +506,7 b' class _gitlfsremote(object):'
492 506 response = b''
493 507 try:
494 508 with contextlib.closing(self.urlopener.open(request)) as res:
509 contentlength = res.info().get(b"content-length")
495 510 ui = self.ui # Shorten debug lines
496 511 if self.ui.debugflag:
497 512 ui.debug(b'Status: %d\n' % res.status)
@@ -503,7 +518,7 b' class _gitlfsremote(object):'
503 518 if action == b'download':
504 519 # If downloading blobs, store downloaded data to local
505 520 # blobstore
506 localstore.download(oid, res)
521 localstore.download(oid, res, contentlength)
507 522 else:
508 523 while True:
509 524 data = res.read(1048576)
@@ -637,7 +652,7 b' class _dummyremote(object):'
637 652 def readbatch(self, pointers, tostore):
638 653 for p in _deduplicate(pointers):
639 654 with self.vfs(p.oid(), b'rb') as fp:
640 tostore.download(p.oid(), fp)
655 tostore.download(p.oid(), fp, None)
641 656
642 657
643 658 class _nullremote(object):
@@ -327,7 +327,7 b' def _processbasictransfer(repo, req, res'
327 327
328 328 statusmessage = hgwebcommon.statusmessage
329 329 try:
330 localstore.download(oid, req.bodyfh)
330 localstore.download(oid, req.bodyfh, req.headers[b'Content-Length'])
331 331 res.status = statusmessage(HTTP_OK if existed else HTTP_CREATED)
332 332 except blobstore.LfsCorruptionError:
333 333 _logexception(req)
@@ -210,7 +210,7 b" though the client doesn't send the blob."
210 210 >
211 211 > store = repo.svfs.lfslocalblobstore
212 212 > class badstore(store.__class__):
213 > def download(self, oid, src):
213 > def download(self, oid, src, contentlength):
214 214 > '''Called in the server to handle reading from the client in a
215 215 > PUT request.'''
216 216 > origread = src.read
@@ -218,7 +218,7 b" though the client doesn't send the blob."
218 218 > # Simulate bad data/checksum failure from the client
219 219 > return b'0' * len(origread(nbytes))
220 220 > src.read = _badread
221 > super(badstore, self).download(oid, src)
221 > super(badstore, self).download(oid, src, contentlength)
222 222 >
223 223 > def _read(self, vfs, oid, verify):
224 224 > '''Called in the server to read data for a GET request, and then
@@ -351,8 +351,8 b' Test a checksum failure during the proce'
351 351 $LOCALIP - - [$ERRDATE$] HG error: (glob)
352 352 $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c': (glob)
353 353 $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob)
354 $LOCALIP - - [$ERRDATE$] HG error: localstore.download(oid, req.bodyfh) (glob)
355 $LOCALIP - - [$ERRDATE$] HG error: super(badstore, self).download(oid, src) (glob)
354 $LOCALIP - - [$ERRDATE$] HG error: localstore.download(oid, req.bodyfh, req.headers[b'Content-Length'])
355 $LOCALIP - - [$ERRDATE$] HG error: super(badstore, self).download(oid, src, contentlength)
356 356 $LOCALIP - - [$ERRDATE$] HG error: _(b'corrupt remote lfs object: %s') % oid (glob)
357 357 $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (no-py3 !)
358 358 $LOCALIP - - [$ERRDATE$] HG error: hgext.lfs.blobstore.LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (py3 !)
General Comments 0
You need to be logged in to leave comments. Login now