http2: send an extra header to signal a non-broken client

Some proxies strip the expect header because they forward requests in
a non-compliant way or as a way to defend against bogus clients.

mercurial/hgweb/hgweb_mod.py

@@ -1,302 +1,303
 # hgweb/hgweb_mod.py - Web interface for a repository.
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 from mercurial import ui, hg, hook, error, encoding, templater
 from common import get_stat, ErrorResponse, permhooks, caching
 from common import HTTP_OK, HTTP_NOT_MODIFIED, HTTP_BAD_REQUEST
 from common import HTTP_NOT_FOUND, HTTP_SERVER_ERROR
 from request import wsgirequest
 import webcommands, protocol, webutil
 
 perms = {
     'changegroup': 'pull',
     'changegroupsubset': 'pull',
     'getbundle': 'pull',
     'stream_out': 'pull',
     'listkeys': 'pull',
     'unbundle': 'push',
     'pushkey': 'push',
 }
 
 class hgweb(object):
     def __init__(self, repo, name=None, baseui=None):
         if isinstance(repo, str):
             if baseui:
                 u = baseui.copy()
             else:
                 u = ui.ui()
             self.repo = hg.repository(u, repo)
         else:
             self.repo = repo
 
         self.repo.ui.setconfig('ui', 'report_untrusted', 'off')
         self.repo.ui.setconfig('ui', 'interactive', 'off')
         hook.redirect(True)
         self.mtime = -1
         self.size = -1
         self.reponame = name
         self.archives = 'zip', 'gz', 'bz2'
         self.stripecount = 1
         # a repo owner may set web.templates in .hg/hgrc to get any file
         # readable by the user running the CGI script
         self.templatepath = self.config('web', 'templates')
 
     # The CGI scripts are often run by a user different from the repo owner.
     # Trust the settings from the .hg/hgrc files by default.
     def config(self, section, name, default=None, untrusted=True):
         return self.repo.ui.config(section, name, default,
                                    untrusted=untrusted)
 
     def configbool(self, section, name, default=False, untrusted=True):
         return self.repo.ui.configbool(section, name, default,
                                        untrusted=untrusted)
 
     def configlist(self, section, name, default=None, untrusted=True):
         return self.repo.ui.configlist(section, name, default,
                                        untrusted=untrusted)
 
     def refresh(self, request=None):
         if request:
             self.repo.ui.environ = request.env
         st = get_stat(self.repo.spath)
         # compare changelog size in addition to mtime to catch
         # rollbacks made less than a second ago
         if st.st_mtime != self.mtime or st.st_size != self.size:
             self.mtime = st.st_mtime
             self.size = st.st_size
             self.repo = hg.repository(self.repo.ui, self.repo.root)
             self.maxchanges = int(self.config("web", "maxchanges", 10))
             self.stripecount = int(self.config("web", "stripes", 1))
             self.maxshortchanges = int(self.config("web", "maxshortchanges", 60))
             self.maxfiles = int(self.config("web", "maxfiles", 10))
             self.allowpull = self.configbool("web", "allowpull", True)
             encoding.encoding = self.config("web", "encoding",
                                             encoding.encoding)
 
     def run(self):
         if not os.environ.get('GATEWAY_INTERFACE', '').startswith("CGI/1."):
             raise RuntimeError("This function is only intended to be "
                                "called while running as a CGI script.")
         import mercurial.hgweb.wsgicgi as wsgicgi
         wsgicgi.launch(self)
 
     def __call__(self, env, respond):
         req = wsgirequest(env, respond)
         return self.run_wsgi(req)
 
     def run_wsgi(self, req):
 
         self.refresh(req)
 
         # work with CGI variables to create coherent structure
         # use SCRIPT_NAME, PATH_INFO and QUERY_STRING as well as our REPO_NAME
 
         req.url = req.env['SCRIPT_NAME']
         if not req.url.endswith('/'):
             req.url += '/'
         if 'REPO_NAME' in req.env:
             req.url += req.env['REPO_NAME'] + '/'
 
         if 'PATH_INFO' in req.env:
             parts = req.env['PATH_INFO'].strip('/').split('/')
             repo_parts = req.env.get('REPO_NAME', '').split('/')
             if parts[:len(repo_parts)] == repo_parts:
                 parts = parts[len(repo_parts):]
             query = '/'.join(parts)
         else:
             query = req.env['QUERY_STRING'].split('&', 1)[0]
             query = query.split(';', 1)[0]
 
         # process this if it's a protocol request
         # protocol bits don't need to create any URLs
         # and the clients always use the old URL structure
 
         cmd = req.form.get('cmd', [''])[0]
         if protocol.iscmd(cmd):
             try:
                 if query:
                     raise ErrorResponse(HTTP_NOT_FOUND)
                 if cmd in perms:
                     self.check_perm(req, perms[cmd])
                 return protocol.call(self.repo, req, cmd)
             except ErrorResponse, inst:
                 # A client that sends unbundle without 100-continue will
                 # break if we respond early.
                 if (cmd == 'unbundle' and
-                    req.env.get('HTTP_EXPECT',
-                                '').lower() != '100-continue'):
+                    (req.env.get('HTTP_EXPECT',
+                                 '').lower() != '100-continue') or
+                    req.env.get('X-HgHttp2', '')):
                     req.drain()
                 req.respond(inst, protocol.HGTYPE)
                 return '0\n%s\n' % inst.message
 
         # translate user-visible url structure to internal structure
 
         args = query.split('/', 2)
         if 'cmd' not in req.form and args and args[0]:
 
             cmd = args.pop(0)
             style = cmd.rfind('-')
             if style != -1:
                 req.form['style'] = [cmd[:style]]
                 cmd = cmd[style + 1:]
 
             # avoid accepting e.g. style parameter as command
             if hasattr(webcommands, cmd):
                 req.form['cmd'] = [cmd]
             else:
                 cmd = ''
 
             if cmd == 'static':
                 req.form['file'] = ['/'.join(args)]
             else:
                 if args and args[0]:
                     node = args.pop(0)
                     req.form['node'] = [node]
                 if args:
                     req.form['file'] = args
 
             ua = req.env.get('HTTP_USER_AGENT', '')
             if cmd == 'rev' and 'mercurial' in ua:
                 req.form['style'] = ['raw']
 
             if cmd == 'archive':
                 fn = req.form['node'][0]
                 for type_, spec in self.archive_specs.iteritems():
                     ext = spec[2]
                     if fn.endswith(ext):
                         req.form['node'] = [fn[:-len(ext)]]
                         req.form['type'] = [type_]
 
         # process the web interface request
 
         try:
             tmpl = self.templater(req)
             ctype = tmpl('mimetype', encoding=encoding.encoding)
             ctype = templater.stringify(ctype)
 
             # check read permissions non-static content
             if cmd != 'static':
                 self.check_perm(req, None)
 
             if cmd == '':
                 req.form['cmd'] = [tmpl.cache['default']]
                 cmd = req.form['cmd'][0]
 
             if self.configbool('web', 'cache', True):
                 caching(self, req) # sets ETag header or raises NOT_MODIFIED
             if cmd not in webcommands.__all__:
                 msg = 'no such method: %s' % cmd
                 raise ErrorResponse(HTTP_BAD_REQUEST, msg)
             elif cmd == 'file' and 'raw' in req.form.get('style', []):
                 self.ctype = ctype
                 content = webcommands.rawfile(self, req, tmpl)
             else:
                 content = getattr(webcommands, cmd)(self, req, tmpl)
                 req.respond(HTTP_OK, ctype)
 
             return content
 
         except error.LookupError, err:
             req.respond(HTTP_NOT_FOUND, ctype)
             msg = str(err)
             if 'manifest' not in msg:
                 msg = 'revision not found: %s' % err.name
             return tmpl('error', error=msg)
         except (error.RepoError, error.RevlogError), inst:
             req.respond(HTTP_SERVER_ERROR, ctype)
             return tmpl('error', error=str(inst))
         except ErrorResponse, inst:
             req.respond(inst, ctype)
             if inst.code == HTTP_NOT_MODIFIED:
                 # Not allowed to return a body on a 304
                 return ['']
             return tmpl('error', error=inst.message)
 
     def templater(self, req):
 
         # determine scheme, port and server name
         # this is needed to create absolute urls
 
         proto = req.env.get('wsgi.url_scheme')
         if proto == 'https':
             proto = 'https'
             default_port = "443"
         else:
             proto = 'http'
             default_port = "80"
 
         port = req.env["SERVER_PORT"]
         port = port != default_port and (":" + port) or ""
         urlbase = '%s://%s%s' % (proto, req.env['SERVER_NAME'], port)
         logourl = self.config("web", "logourl", "http://mercurial.selenic.com/")
         staticurl = self.config("web", "staticurl") or req.url + 'static/'
         if not staticurl.endswith('/'):
             staticurl += '/'
 
         # some functions for the templater
 
         def header(**map):
             yield tmpl('header', encoding=encoding.encoding, **map)
 
         def footer(**map):
             yield tmpl("footer", **map)
 
         def motd(**map):
             yield self.config("web", "motd", "")
 
         # figure out which style to use
 
         vars = {}
         styles = (
             req.form.get('style', [None])[0],
             self.config('web', 'style'),
             'paper',
         )
         style, mapfile = templater.stylemap(styles, self.templatepath)
         if style == styles[0]:
             vars['style'] = style
 
         start = req.url[-1] == '?' and '&' or '?'
         sessionvars = webutil.sessionvars(vars, start)
 
         if not self.reponame:
             self.reponame = (self.config("web", "name")
                              or req.env.get('REPO_NAME')
                              or req.url.strip('/') or self.repo.root)
 
         # create the templater
 
         tmpl = templater.templater(mapfile,
                                    defaults={"url": req.url,
                                              "logourl": logourl,
                                              "staticurl": staticurl,
                                              "urlbase": urlbase,
                                              "repo": self.reponame,
                                              "header": header,
                                              "footer": footer,
                                              "motd": motd,
                                              "sessionvars": sessionvars
                                             })
         return tmpl
 
     def archivelist(self, nodeid):
         allowed = self.configlist("web", "allow_archive")
         for i, spec in self.archive_specs.iteritems():
             if i in allowed or self.configbool("web", "allow" + i):
                 yield {"type" : i, "extension" : spec[2], "node" : nodeid}
 
     archive_specs = {
         'bz2': ('application/x-bzip2', 'tbz2', '.tar.bz2', None),
         'gz': ('application/x-gzip', 'tgz', '.tar.gz', None),
         'zip': ('application/zip', 'zip', '.zip', None),
         }
 
     def check_perm(self, req, op):
         for hook in permhooks:
             hook(self, req, op)

mercurial/httprepo.py

@@ -1,241 +1,242
 # httprepo.py - HTTP repository proxy classes for mercurial
 #
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from node import nullid
 from i18n import _
 import changegroup, statichttprepo, error, httpconnection, url, util, wireproto
 import os, urllib, urllib2, zlib, httplib
 import errno, socket
 
 def zgenerator(f):
     zd = zlib.decompressobj()
     try:
         for chunk in util.filechunkiter(f):
             while chunk:
                 yield zd.decompress(chunk, 2**18)
                 chunk = zd.unconsumed_tail
     except httplib.HTTPException:
         raise IOError(None, _('connection ended unexpectedly'))
     yield zd.flush()
 
 class httprepository(wireproto.wirerepository):
     def __init__(self, ui, path):
         self.path = path
         self.caps = None
         self.handler = None
         u = util.url(path)
         if u.query or u.fragment:
             raise util.Abort(_('unsupported URL component: "%s"') %
                              (u.query or u.fragment))
 
         # urllib cannot handle URLs with embedded user or passwd
         self._url, authinfo = u.authinfo()
 
         self.ui = ui
         self.ui.debug('using %s\n' % self._url)
 
         self.urlopener = url.opener(ui, authinfo)
 
     def __del__(self):
         for h in self.urlopener.handlers:
             h.close()
             if hasattr(h, "close_all"):
                 h.close_all()
 
     def url(self):
         return self.path
 
     # look up capabilities only when needed
 
     def _fetchcaps(self):
         self.caps = set(self._call('capabilities').split())
 
     def get_caps(self):
         if self.caps is None:
             try:
                 self._fetchcaps()
             except error.RepoError:
                 self.caps = set()
             self.ui.debug('capabilities: %s\n' %
                           (' '.join(self.caps or ['none'])))
         return self.caps
 
     capabilities = property(get_caps)
 
     def lock(self):
         raise util.Abort(_('operation not supported over http'))
 
     def _callstream(self, cmd, **args):
         if cmd == 'pushkey':
             args['data'] = ''
         data = args.pop('data', None)
         headers = args.pop('headers', {})
 
         if data and self.ui.configbool('ui', 'usehttp2', False):
             headers['Expect'] = '100-Continue'
+            headers['X-HgHttp2'] = '1'
 
         self.ui.debug("sending %s command\n" % cmd)
         q = [('cmd', cmd)]
         headersize = 0
         if len(args) > 0:
             httpheader = self.capable('httpheader')
             if httpheader:
                 headersize = int(httpheader.split(',')[0])
         if headersize > 0:
             # The headers can typically carry more data than the URL.
             encargs = urllib.urlencode(sorted(args.items()))
             headerfmt = 'X-HgArg-%s'
             contentlen = headersize - len(headerfmt % '000' + ': \r\n')
             headernum = 0
             for i in xrange(0, len(encargs), contentlen):
                 headernum += 1
                 header = headerfmt % str(headernum)
                 headers[header] = encargs[i:i + contentlen]
             varyheaders = [headerfmt % str(h) for h in range(1, headernum + 1)]
             headers['Vary'] = ','.join(varyheaders)
         else:
             q += sorted(args.items())
         qs = '?%s' % urllib.urlencode(q)
         cu = "%s%s" % (self._url, qs)
         req = urllib2.Request(cu, data, headers)
         if data is not None:
             # len(data) is broken if data doesn't fit into Py_ssize_t
             # add the header ourself to avoid OverflowError
             size = data.__len__()
             self.ui.debug("sending %s bytes\n" % size)
             req.add_unredirected_header('Content-Length', '%d' % size)
         try:
             resp = self.urlopener.open(req)
         except urllib2.HTTPError, inst:
             if inst.code == 401:
                 raise util.Abort(_('authorization failed'))
             raise
         except httplib.HTTPException, inst:
             self.ui.debug('http error while sending %s command\n' % cmd)
             self.ui.traceback()
             raise IOError(None, inst)
         except IndexError:
             # this only happens with Python 2.3, later versions raise URLError
             raise util.Abort(_('http error, possibly caused by proxy setting'))
         # record the url we got redirected to
         resp_url = resp.geturl()
         if resp_url.endswith(qs):
             resp_url = resp_url[:-len(qs)]
         if self._url.rstrip('/') != resp_url.rstrip('/'):
             if not self.ui.quiet:
                 self.ui.warn(_('real URL is %s\n') % resp_url)
         self._url = resp_url
         try:
             proto = resp.getheader('content-type')
         except AttributeError:
             proto = resp.headers.get('content-type', '')
 
         safeurl = util.hidepassword(self._url)
         # accept old "text/plain" and "application/hg-changegroup" for now
         if not (proto.startswith('application/mercurial-') or
                 proto.startswith('text/plain') or
                 proto.startswith('application/hg-changegroup')):
             self.ui.debug("requested URL: '%s'\n" % util.hidepassword(cu))
             raise error.RepoError(
                 _("'%s' does not appear to be an hg repository:\n"
                   "---%%<--- (%s)\n%s\n---%%<---\n")
                 % (safeurl, proto or 'no content-type', resp.read()))
 
         if proto.startswith('application/mercurial-'):
             try:
                 version = proto.split('-', 1)[1]
                 version_info = tuple([int(n) for n in version.split('.')])
             except ValueError:
                 raise error.RepoError(_("'%s' sent a broken Content-Type "
                                         "header (%s)") % (safeurl, proto))
             if version_info > (0, 1):
                 raise error.RepoError(_("'%s' uses newer protocol %s") %
                                       (safeurl, version))
 
         return resp
 
     def _call(self, cmd, **args):
         fp = self._callstream(cmd, **args)
         try:
             return fp.read()
         finally:
             # if using keepalive, allow connection to be reused
             fp.close()
 
     def _callpush(self, cmd, cg, **args):
         # have to stream bundle to a temp file because we do not have
         # http 1.1 chunked transfer.
 
         types = self.capable('unbundle')
         try:
             types = types.split(',')
         except AttributeError:
             # servers older than d1b16a746db6 will send 'unbundle' as a
             # boolean capability. They only support headerless/uncompressed
             # bundles.
             types = [""]
         for x in types:
             if x in changegroup.bundletypes:
                 type = x
                 break
 
         tempname = changegroup.writebundle(cg, None, type)
         fp = httpconnection.httpsendfile(self.ui, tempname, "rb")
         headers = {'Content-Type': 'application/mercurial-0.1'}
 
         try:
             try:
                 r = self._call(cmd, data=fp, headers=headers, **args)
                 vals = r.split('\n', 1)
                 if len(vals) < 2:
                     raise error.ResponseError(_("unexpected response:"), r)
                 return vals
             except socket.error, err:
                 if err.args[0] in (errno.ECONNRESET, errno.EPIPE):
                     raise util.Abort(_('push failed: %s') % err.args[1])
                 raise util.Abort(err.args[1])
         finally:
             fp.close()
             os.unlink(tempname)
 
     def _abort(self, exception):
         raise exception
 
     def _decompress(self, stream):
         return util.chunkbuffer(zgenerator(stream))
 
 class httpsrepository(httprepository):
     def __init__(self, ui, path):
         if not url.has_https:
             raise util.Abort(_('Python support for SSL and HTTPS '
                                'is not installed'))
         httprepository.__init__(self, ui, path)
 
 def instance(ui, path, create):
     if create:
         raise util.Abort(_('cannot create new http repository'))
     try:
         if path.startswith('https:'):
             inst = httpsrepository(ui, path)
         else:
             inst = httprepository(ui, path)
         try:
             # Try to do useful work when checking compatibility.
             # Usually saves a roundtrip since we want the caps anyway.
             inst._fetchcaps()
         except error.RepoError:
             # No luck, try older compatibility check.
             inst.between([(nullid, nullid)])
         return inst
     except error.RepoError, httpexception:
         try:
             r = statichttprepo.instance(ui, "static-" + path, create)
             ui.note('(falling back to static-http)\n')
             return r
         except error.RepoError:
             raise httpexception # use the original http RepoError instead