##// END OF EJS Templates
lfs: check content length after downloading content...
Matt Harbison -
r44929:0ee0a3f6 default
parent child Browse files
Show More
@@ -155,15 +155,29 b' class local(object):'
155
155
156 return self.vfs(oid, b'rb')
156 return self.vfs(oid, b'rb')
157
157
158 def download(self, oid, src):
158 def download(self, oid, src, content_length):
159 """Read the blob from the remote source in chunks, verify the content,
159 """Read the blob from the remote source in chunks, verify the content,
160 and write to this local blobstore."""
160 and write to this local blobstore."""
161 sha256 = hashlib.sha256()
161 sha256 = hashlib.sha256()
162 size = 0
162
163
163 with self.vfs(oid, b'wb', atomictemp=True) as fp:
164 with self.vfs(oid, b'wb', atomictemp=True) as fp:
164 for chunk in util.filechunkiter(src, size=1048576):
165 for chunk in util.filechunkiter(src, size=1048576):
165 fp.write(chunk)
166 fp.write(chunk)
166 sha256.update(chunk)
167 sha256.update(chunk)
168 size += len(chunk)
169
170 # If the server advertised a length longer than what we actually
171 # received, then we should expect that the server crashed while
172 # producing the response (but the server has no way of telling us
173 # that), and we really don't need to try to write the response to
174 # the localstore, because it's not going to match the expected.
175 if content_length is not None and int(content_length) != size:
176 msg = (
177 b"Response length (%s) does not match Content-Length "
178 b"header (%d): likely server-side crash"
179 )
180 raise LfsRemoteError(_(msg) % (size, int(content_length)))
167
181
168 realoid = node.hex(sha256.digest())
182 realoid = node.hex(sha256.digest())
169 if realoid != oid:
183 if realoid != oid:
@@ -492,6 +506,7 b' class _gitlfsremote(object):'
492 response = b''
506 response = b''
493 try:
507 try:
494 with contextlib.closing(self.urlopener.open(request)) as res:
508 with contextlib.closing(self.urlopener.open(request)) as res:
509 contentlength = res.info().get(b"content-length")
495 ui = self.ui # Shorten debug lines
510 ui = self.ui # Shorten debug lines
496 if self.ui.debugflag:
511 if self.ui.debugflag:
497 ui.debug(b'Status: %d\n' % res.status)
512 ui.debug(b'Status: %d\n' % res.status)
@@ -503,7 +518,7 b' class _gitlfsremote(object):'
503 if action == b'download':
518 if action == b'download':
504 # If downloading blobs, store downloaded data to local
519 # If downloading blobs, store downloaded data to local
505 # blobstore
520 # blobstore
506 localstore.download(oid, res)
521 localstore.download(oid, res, contentlength)
507 else:
522 else:
508 while True:
523 while True:
509 data = res.read(1048576)
524 data = res.read(1048576)
@@ -637,7 +652,7 b' class _dummyremote(object):'
637 def readbatch(self, pointers, tostore):
652 def readbatch(self, pointers, tostore):
638 for p in _deduplicate(pointers):
653 for p in _deduplicate(pointers):
639 with self.vfs(p.oid(), b'rb') as fp:
654 with self.vfs(p.oid(), b'rb') as fp:
640 tostore.download(p.oid(), fp)
655 tostore.download(p.oid(), fp, None)
641
656
642
657
643 class _nullremote(object):
658 class _nullremote(object):
@@ -327,7 +327,7 b' def _processbasictransfer(repo, req, res'
327
327
328 statusmessage = hgwebcommon.statusmessage
328 statusmessage = hgwebcommon.statusmessage
329 try:
329 try:
330 localstore.download(oid, req.bodyfh)
330 localstore.download(oid, req.bodyfh, req.headers[b'Content-Length'])
331 res.status = statusmessage(HTTP_OK if existed else HTTP_CREATED)
331 res.status = statusmessage(HTTP_OK if existed else HTTP_CREATED)
332 except blobstore.LfsCorruptionError:
332 except blobstore.LfsCorruptionError:
333 _logexception(req)
333 _logexception(req)
@@ -210,7 +210,7 b" though the client doesn't send the blob."
210 >
210 >
211 > store = repo.svfs.lfslocalblobstore
211 > store = repo.svfs.lfslocalblobstore
212 > class badstore(store.__class__):
212 > class badstore(store.__class__):
213 > def download(self, oid, src):
213 > def download(self, oid, src, contentlength):
214 > '''Called in the server to handle reading from the client in a
214 > '''Called in the server to handle reading from the client in a
215 > PUT request.'''
215 > PUT request.'''
216 > origread = src.read
216 > origread = src.read
@@ -218,7 +218,7 b" though the client doesn't send the blob."
218 > # Simulate bad data/checksum failure from the client
218 > # Simulate bad data/checksum failure from the client
219 > return b'0' * len(origread(nbytes))
219 > return b'0' * len(origread(nbytes))
220 > src.read = _badread
220 > src.read = _badread
221 > super(badstore, self).download(oid, src)
221 > super(badstore, self).download(oid, src, contentlength)
222 >
222 >
223 > def _read(self, vfs, oid, verify):
223 > def _read(self, vfs, oid, verify):
224 > '''Called in the server to read data for a GET request, and then
224 > '''Called in the server to read data for a GET request, and then
@@ -351,8 +351,8 b' Test a checksum failure during the proce'
351 $LOCALIP - - [$ERRDATE$] HG error: (glob)
351 $LOCALIP - - [$ERRDATE$] HG error: (glob)
352 $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c': (glob)
352 $LOCALIP - - [$ERRDATE$] HG error: Exception happened while processing request '/.hg/lfs/objects/b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c': (glob)
353 $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob)
353 $LOCALIP - - [$ERRDATE$] HG error: Traceback (most recent call last): (glob)
354 $LOCALIP - - [$ERRDATE$] HG error: localstore.download(oid, req.bodyfh) (glob)
354 $LOCALIP - - [$ERRDATE$] HG error: localstore.download(oid, req.bodyfh, req.headers[b'Content-Length'])
355 $LOCALIP - - [$ERRDATE$] HG error: super(badstore, self).download(oid, src) (glob)
355 $LOCALIP - - [$ERRDATE$] HG error: super(badstore, self).download(oid, src, contentlength)
356 $LOCALIP - - [$ERRDATE$] HG error: _(b'corrupt remote lfs object: %s') % oid (glob)
356 $LOCALIP - - [$ERRDATE$] HG error: _(b'corrupt remote lfs object: %s') % oid (glob)
357 $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (no-py3 !)
357 $LOCALIP - - [$ERRDATE$] HG error: LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (no-py3 !)
358 $LOCALIP - - [$ERRDATE$] HG error: hgext.lfs.blobstore.LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (py3 !)
358 $LOCALIP - - [$ERRDATE$] HG error: hgext.lfs.blobstore.LfsCorruptionError: corrupt remote lfs object: b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c (py3 !)
General Comments 0
You need to be logged in to leave comments. Login now