lfs: update the HTTP status codes in error cases

I'm not bothering with validating PUT requests (for now), because the spec
doesn't explicitly call out a Content-Type (though the example transcript does
use the sensible 'application/octet-stream').

hgext/lfs/wireprotolfsserver.py

@@ -1,320 +1,327 b''
 # wireprotolfsserver.py - lfs protocol server side implementation
 #
 # Copyright 2018 Matt Harbison <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import datetime
 import errno
 import json
 import traceback
 
 from mercurial.hgweb import (
     common as hgwebcommon,
 )
 
 from mercurial import (
     pycompat,
 )
 
 from . import blobstore
 
 HTTP_OK = hgwebcommon.HTTP_OK
 HTTP_CREATED = hgwebcommon.HTTP_CREATED
 HTTP_BAD_REQUEST = hgwebcommon.HTTP_BAD_REQUEST
 HTTP_NOT_FOUND = hgwebcommon.HTTP_NOT_FOUND
+HTTP_METHOD_NOT_ALLOWED = hgwebcommon.HTTP_METHOD_NOT_ALLOWED
+HTTP_NOT_ACCEPTABLE = hgwebcommon.HTTP_NOT_ACCEPTABLE
+HTTP_UNSUPPORTED_MEDIA_TYPE = hgwebcommon.HTTP_UNSUPPORTED_MEDIA_TYPE
 
 def handlewsgirequest(orig, rctx, req, res, checkperm):
     """Wrap wireprotoserver.handlewsgirequest() to possibly process an LFS
     request if it is left unprocessed by the wrapped method.
     """
     if orig(rctx, req, res, checkperm):
         return True
 
     if not rctx.repo.ui.configbool('experimental', 'lfs.serve'):
         return False
 
     if not req.dispatchpath:
         return False
 
     try:
         if req.dispatchpath == b'.git/info/lfs/objects/batch':
             checkperm(rctx, req, 'pull')
             return _processbatchrequest(rctx.repo, req, res)
         # TODO: reserve and use a path in the proposed http wireprotocol /api/
         #       namespace?
         elif req.dispatchpath.startswith(b'.hg/lfs/objects'):
             return _processbasictransfer(rctx.repo, req, res,
                                          lambda perm:
                                                 checkperm(rctx, req, perm))
         return False
     except hgwebcommon.ErrorResponse as e:
         # XXX: copied from the handler surrounding wireprotoserver._callhttp()
         #      in the wrapped function.  Should this be moved back to hgweb to
         #      be a common handler?
         for k, v in e.headers:
             res.headers[k] = v
         res.status = hgwebcommon.statusmessage(e.code, pycompat.bytestr(e))
         res.setbodybytes(b'0\n%s\n' % pycompat.bytestr(e))
         return True
 
 def _sethttperror(res, code, message=None):
     res.status = hgwebcommon.statusmessage(code, message=message)
     res.headers[b'Content-Type'] = b'text/plain; charset=utf-8'
     res.setbodybytes(b'')
 
 def _logexception(req):
     """Write information about the current exception to wsgi.errors."""
     tb = pycompat.sysbytes(traceback.format_exc())
     errorlog = req.rawenv[r'wsgi.errors']
 
     uri = b''
     if req.apppath:
         uri += req.apppath
     uri += b'/' + req.dispatchpath
 
     errorlog.write(b"Exception happened while processing request '%s':\n%s" %
                    (uri, tb))
 
 def _processbatchrequest(repo, req, res):
     """Handle a request for the Batch API, which is the gateway to granting file
     access.
 
     https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
     """
 
     # Mercurial client request:
     #
     #   HOST: localhost:$HGPORT
     #   ACCEPT: application/vnd.git-lfs+json
     #   ACCEPT-ENCODING: identity
     #   USER-AGENT: git-lfs/2.3.4 (Mercurial 4.5.2+1114-f48b9754f04c+20180316)
     #   Content-Length: 125
     #   Content-Type: application/vnd.git-lfs+json
     #
     #   {
     #     "objects": [
     #       {
     #         "oid": "31cf...8e5b"
     #         "size": 12
     #       }
     #     ]
     #     "operation": "upload"
     #  }
 
-    if (req.method != b'POST'
-        or req.headers[b'Content-Type'] != b'application/vnd.git-lfs+json'
-        or req.headers[b'Accept'] != b'application/vnd.git-lfs+json'):
-        # TODO: figure out what the proper handling for a bad request to the
-        #       Batch API is.
-        _sethttperror(res, HTTP_BAD_REQUEST, b'Invalid Batch API request')
+    if req.method != b'POST':
+        _sethttperror(res, HTTP_METHOD_NOT_ALLOWED)
+        return True
+
+    if req.headers[b'Content-Type'] != b'application/vnd.git-lfs+json':
+        _sethttperror(res, HTTP_UNSUPPORTED_MEDIA_TYPE)
+        return True
+
+    if req.headers[b'Accept'] != b'application/vnd.git-lfs+json':
+        _sethttperror(res, HTTP_NOT_ACCEPTABLE)
         return True
 
     # XXX: specify an encoding?
     lfsreq = json.loads(req.bodyfh.read())
 
     # If no transfer handlers are explicitly requested, 'basic' is assumed.
     if 'basic' not in lfsreq.get('transfers', ['basic']):
         _sethttperror(res, HTTP_BAD_REQUEST,
                       b'Only the basic LFS transfer handler is supported')
         return True
 
     operation = lfsreq.get('operation')
     if operation not in ('upload', 'download'):
         _sethttperror(res, HTTP_BAD_REQUEST,
                       b'Unsupported LFS transfer operation: %s' % operation)
         return True
 
     localstore = repo.svfs.lfslocalblobstore
 
     objects = [p for p in _batchresponseobjects(req, lfsreq.get('objects', []),
                                                 operation, localstore)]
 
     rsp = {
         'transfer': 'basic',
         'objects': objects,
     }
 
     res.status = hgwebcommon.statusmessage(HTTP_OK)
     res.headers[b'Content-Type'] = b'application/vnd.git-lfs+json'
     res.setbodybytes(pycompat.bytestr(json.dumps(rsp)))
 
     return True
 
 def _batchresponseobjects(req, objects, action, store):
     """Yield one dictionary of attributes for the Batch API response for each
     object in the list.
 
     req: The parsedrequest for the Batch API request
     objects: The list of objects in the Batch API object request list
     action: 'upload' or 'download'
     store: The local blob store for servicing requests"""
 
     # Successful lfs-test-server response to solict an upload:
     # {
     #    u'objects': [{
     #       u'size': 12,
     #       u'oid': u'31cf...8e5b',
     #       u'actions': {
     #           u'upload': {
     #               u'href': u'http://localhost:$HGPORT/objects/31cf...8e5b',
     #               u'expires_at': u'0001-01-01T00:00:00Z',
     #               u'header': {
     #                   u'Accept': u'application/vnd.git-lfs'
     #               }
     #           }
     #       }
     #    }]
     # }
 
     # TODO: Sort out the expires_at/expires_in/authenticated keys.
 
     for obj in objects:
         # Convert unicode to ASCII to create a filesystem path
         oid = obj.get('oid').encode('ascii')
         rsp = {
             'oid': oid,
             'size': obj.get('size'),  # XXX: should this check the local size?
             #'authenticated': True,
         }
 
         exists = True
         verifies = False
 
         # Verify an existing file on the upload request, so that the client is
         # solicited to re-upload if it corrupt locally.  Download requests are
         # also verified, so the error can be flagged in the Batch API response.
         # (Maybe we can use this to short circuit the download for `hg verify`,
         # IFF the client can assert that the remote end is an hg server.)
         # Otherwise, it's potentially overkill on download, since it is also
         # verified as the file is streamed to the caller.
         try:
             verifies = store.verify(oid)
         except IOError as inst:
             if inst.errno != errno.ENOENT:
                 _logexception(req)
 
                 rsp['error'] = {
                     'code': 500,
                     'message': inst.strerror or 'Internal Server Server'
                 }
                 yield rsp
                 continue
 
             exists = False
 
         # Items are always listed for downloads.  They are dropped for uploads
         # IFF they already exist locally.
         if action == 'download':
             if not exists:
                 rsp['error'] = {
                     'code': 404,
                     'message': "The object does not exist"
                 }
                 yield rsp
                 continue
 
             elif not verifies:
                 rsp['error'] = {
                     'code': 422,   # XXX: is this the right code?
                     'message': "The object is corrupt"
                 }
                 yield rsp
                 continue
 
         elif verifies:
             yield rsp  # Skip 'actions': already uploaded
             continue
 
         expiresat = datetime.datetime.now() + datetime.timedelta(minutes=10)
 
         rsp['actions'] = {
             '%s' % action: {
                 'href': '%s%s/.hg/lfs/objects/%s'
                     % (req.baseurl, req.apppath, oid),
                 # datetime.isoformat() doesn't include the 'Z' suffix
                 "expires_at": expiresat.strftime('%Y-%m-%dT%H:%M:%SZ'),
                 'header': {
                     # The spec doesn't mention the Accept header here, but avoid
                     # a gratuitous deviation from lfs-test-server in the test
                     # output.
                     'Accept': 'application/vnd.git-lfs'
                 }
             }
         }
 
         yield rsp
 
 def _processbasictransfer(repo, req, res, checkperm):
     """Handle a single file upload (PUT) or download (GET) action for the Basic
     Transfer Adapter.
 
     After determining if the request is for an upload or download, the access
     must be checked by calling ``checkperm()`` with either 'pull' or 'upload'
     before accessing the files.
 
     https://github.com/git-lfs/git-lfs/blob/master/docs/api/basic-transfers.md
     """
 
     method = req.method
     oid = req.dispatchparts[-1]
     localstore = repo.svfs.lfslocalblobstore
 
     if len(req.dispatchparts) != 4:
         _sethttperror(res, HTTP_NOT_FOUND)
         return True
 
     if method == b'PUT':
         checkperm('upload')
 
         # TODO: verify Content-Type?
 
         existed = localstore.has(oid)
 
         # TODO: how to handle timeouts?  The body proxy handles limiting to
         #       Content-Length, but what happens if a client sends less than it
         #       says it will?
 
         statusmessage = hgwebcommon.statusmessage
         try:
             localstore.download(oid, req.bodyfh)
             res.status = statusmessage(HTTP_OK if existed else HTTP_CREATED)
         except blobstore.LfsCorruptionError:
             _logexception(req)
 
             # XXX: Is this the right code?
             res.status = statusmessage(422, b'corrupt blob')
 
         # There's no payload here, but this is the header that lfs-test-server
         # sends back.  This eliminates some gratuitous test output conditionals.
         res.headers[b'Content-Type'] = b'text/plain; charset=utf-8'
         res.setbodybytes(b'')
 
         return True
     elif method == b'GET':
         checkperm('pull')
 
         res.status = hgwebcommon.statusmessage(HTTP_OK)
         res.headers[b'Content-Type'] = b'application/octet-stream'
 
         try:
             # TODO: figure out how to send back the file in chunks, instead of
             #       reading the whole thing.  (Also figure out how to send back
             #       an error status if an IOError occurs after a partial write
             #       in that case.  Here, everything is read before starting.)
             res.setbodybytes(localstore.read(oid))
         except blobstore.LfsCorruptionError:
             _logexception(req)
 
             # XXX: Is this the right code?
             res.status = hgwebcommon.statusmessage(422, b'corrupt blob')
             res.setbodybytes(b'')
 
         return True
     else:
-        _sethttperror(res, HTTP_BAD_REQUEST,
+        _sethttperror(res, HTTP_METHOD_NOT_ALLOWED,
                       message=b'Unsupported LFS transfer method: %s' % method)
         return True

mercurial/hgweb/common.py

@@ -1,258 +1,260 b''
 # hgweb/common.py - Utility functions needed by hgweb_mod and hgwebdir_mod
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import base64
 import errno
 import mimetypes
 import os
 import stat
 
 from .. import (
     encoding,
     pycompat,
     util,
 )
 
 httpserver = util.httpserver
 
 HTTP_OK = 200
 HTTP_CREATED = 201
 HTTP_NOT_MODIFIED = 304
 HTTP_BAD_REQUEST = 400
 HTTP_UNAUTHORIZED = 401
 HTTP_FORBIDDEN = 403
 HTTP_NOT_FOUND = 404
 HTTP_METHOD_NOT_ALLOWED = 405
+HTTP_NOT_ACCEPTABLE = 406
+HTTP_UNSUPPORTED_MEDIA_TYPE = 415
 HTTP_SERVER_ERROR = 500
 
 
 def ismember(ui, username, userlist):
     """Check if username is a member of userlist.
 
     If userlist has a single '*' member, all users are considered members.
     Can be overridden by extensions to provide more complex authorization
     schemes.
     """
     return userlist == ['*'] or username in userlist
 
 def checkauthz(hgweb, req, op):
     '''Check permission for operation based on request data (including
     authentication info). Return if op allowed, else raise an ErrorResponse
     exception.'''
 
     user = req.remoteuser
 
     deny_read = hgweb.configlist('web', 'deny_read')
     if deny_read and (not user or ismember(hgweb.repo.ui, user, deny_read)):
         raise ErrorResponse(HTTP_UNAUTHORIZED, 'read not authorized')
 
     allow_read = hgweb.configlist('web', 'allow_read')
     if allow_read and (not ismember(hgweb.repo.ui, user, allow_read)):
         raise ErrorResponse(HTTP_UNAUTHORIZED, 'read not authorized')
 
     if op == 'pull' and not hgweb.allowpull:
         raise ErrorResponse(HTTP_UNAUTHORIZED, 'pull not authorized')
     elif op == 'pull' or op is None: # op is None for interface requests
         return
 
     # Allow LFS uploading via PUT requests
     if op == 'upload':
         if req.method != 'PUT':
             msg = 'upload requires PUT request'
             raise ErrorResponse(HTTP_METHOD_NOT_ALLOWED, msg)
     # enforce that you can only push using POST requests
     elif req.method != 'POST':
         msg = 'push requires POST request'
         raise ErrorResponse(HTTP_METHOD_NOT_ALLOWED, msg)
 
     # require ssl by default for pushing, auth info cannot be sniffed
     # and replayed
     if hgweb.configbool('web', 'push_ssl') and req.urlscheme != 'https':
         raise ErrorResponse(HTTP_FORBIDDEN, 'ssl required')
 
     deny = hgweb.configlist('web', 'deny_push')
     if deny and (not user or ismember(hgweb.repo.ui, user, deny)):
         raise ErrorResponse(HTTP_UNAUTHORIZED, 'push not authorized')
 
     allow = hgweb.configlist('web', 'allow-push')
     if not (allow and ismember(hgweb.repo.ui, user, allow)):
         raise ErrorResponse(HTTP_UNAUTHORIZED, 'push not authorized')
 
 # Hooks for hgweb permission checks; extensions can add hooks here.
 # Each hook is invoked like this: hook(hgweb, request, operation),
 # where operation is either read, pull, push or upload. Hooks should either
 # raise an ErrorResponse exception, or just return.
 #
 # It is possible to do both authentication and authorization through
 # this.
 permhooks = [checkauthz]
 
 
 class ErrorResponse(Exception):
     def __init__(self, code, message=None, headers=None):
         if message is None:
             message = _statusmessage(code)
         Exception.__init__(self, pycompat.sysstr(message))
         self.code = code
         if headers is None:
             headers = []
         self.headers = headers
 
 class continuereader(object):
     """File object wrapper to handle HTTP 100-continue.
 
     This is used by servers so they automatically handle Expect: 100-continue
     request headers. On first read of the request body, the 100 Continue
     response is sent. This should trigger the client into actually sending
     the request body.
     """
     def __init__(self, f, write):
         self.f = f
         self._write = write
         self.continued = False
 
     def read(self, amt=-1):
         if not self.continued:
             self.continued = True
             self._write('HTTP/1.1 100 Continue\r\n\r\n')
         return self.f.read(amt)
 
     def __getattr__(self, attr):
         if attr in ('close', 'readline', 'readlines', '__iter__'):
             return getattr(self.f, attr)
         raise AttributeError
 
 def _statusmessage(code):
     responses = httpserver.basehttprequesthandler.responses
     return responses.get(code, ('Error', 'Unknown error'))[0]
 
 def statusmessage(code, message=None):
     return '%d %s' % (code, message or _statusmessage(code))
 
 def get_stat(spath, fn):
     """stat fn if it exists, spath otherwise"""
     cl_path = os.path.join(spath, fn)
     if os.path.exists(cl_path):
         return os.stat(cl_path)
     else:
         return os.stat(spath)
 
 def get_mtime(spath):
     return get_stat(spath, "00changelog.i")[stat.ST_MTIME]
 
 def ispathsafe(path):
     """Determine if a path is safe to use for filesystem access."""
     parts = path.split('/')
     for part in parts:
         if (part in ('', pycompat.oscurdir, pycompat.ospardir) or
             pycompat.ossep in part or
             pycompat.osaltsep is not None and pycompat.osaltsep in part):
             return False
 
     return True
 
 def staticfile(directory, fname, res):
     """return a file inside directory with guessed Content-Type header
 
     fname always uses '/' as directory separator and isn't allowed to
     contain unusual path components.
     Content-Type is guessed using the mimetypes module.
     Return an empty string if fname is illegal or file not found.
 
     """
     if not ispathsafe(fname):
         return
 
     fpath = os.path.join(*fname.split('/'))
     if isinstance(directory, str):
         directory = [directory]
     for d in directory:
         path = os.path.join(d, fpath)
         if os.path.exists(path):
             break
     try:
         os.stat(path)
         ct = mimetypes.guess_type(pycompat.fsdecode(path))[0] or "text/plain"
         with open(path, 'rb') as fh:
             data = fh.read()
 
         res.headers['Content-Type'] = ct
         res.setbodybytes(data)
         return res
     except TypeError:
         raise ErrorResponse(HTTP_SERVER_ERROR, 'illegal filename')
     except OSError as err:
         if err.errno == errno.ENOENT:
             raise ErrorResponse(HTTP_NOT_FOUND)
         else:
             raise ErrorResponse(HTTP_SERVER_ERROR,
                                 encoding.strtolocal(err.strerror))
 
 def paritygen(stripecount, offset=0):
     """count parity of horizontal stripes for easier reading"""
     if stripecount and offset:
         # account for offset, e.g. due to building the list in reverse
         count = (stripecount + offset) % stripecount
         parity = (stripecount + offset) // stripecount & 1
     else:
         count = 0
         parity = 0
     while True:
         yield parity
         count += 1
         if stripecount and count >= stripecount:
             parity = 1 - parity
             count = 0
 
 def get_contact(config):
     """Return repo contact information or empty string.
 
     web.contact is the primary source, but if that is not set, try
     ui.username or $EMAIL as a fallback to display something useful.
     """
     return (config("web", "contact") or
             config("ui", "username") or
             encoding.environ.get("EMAIL") or "")
 
 def cspvalues(ui):
     """Obtain the Content-Security-Policy header and nonce value.
 
     Returns a 2-tuple of the CSP header value and the nonce value.
 
     First value is ``None`` if CSP isn't enabled. Second value is ``None``
     if CSP isn't enabled or if the CSP header doesn't need a nonce.
     """
     # Without demandimport, "import uuid" could have an immediate side-effect
     # running "ldconfig" on Linux trying to find libuuid.
     # With Python <= 2.7.12, that "ldconfig" is run via a shell and the shell
     # may pollute the terminal with:
     #
     #   shell-init: error retrieving current directory: getcwd: cannot access
     #   parent directories: No such file or directory
     #
     # Python >= 2.7.13 has fixed it by running "ldconfig" directly without a
     # shell (hg changeset a09ae70f3489).
     #
     # Moved "import uuid" from here so it's executed after we know we have
     # a sane cwd (i.e. after dispatch.py cwd check).
     #
     # We can move it back once we no longer need Python <= 2.7.12 support.
     import uuid
 
     # Don't allow untrusted CSP setting since it be disable protections
     # from a trusted/global source.
     csp = ui.config('web', 'csp', untrusted=False)
     nonce = None
 
     if csp and '%nonce%' in csp:
         nonce = base64.urlsafe_b64encode(uuid.uuid4().bytes).rstrip('=')
         csp = csp.replace('%nonce%', nonce)
 
     return csp, nonce