hgweb: teach WSGI parser about query strings

Currently, req.form uses cgi.parse() to populate form data. Depending
on the request, form data can come from POST multipart/form-data,
application/x-www-form-urlencoded, or the URL query string.

Putting all these things into one data structure makes it difficult
to reason about how exactly parameters got to the request. It can
lead to wonkiness such as pulling parameters from both the URL and
POST data.

This commit teaches our WSGI request parser about argument data
in query strings. We populate fields containing the query string
data and only the query string data so it can't be confused with
POST data.

Differential Revision: https://phab.mercurial-scm.org/D2737

mercurial/hgweb/request.py

@@ -1,279 +1,296 b''
 # hgweb/request.py - An http request from either CGI or the standalone server.
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import cgi
 import errno
 import socket
 #import wsgiref.validate
 
 from .common import (
     ErrorResponse,
     HTTP_NOT_MODIFIED,
     statusmessage,
 )
 
 from ..thirdparty import (
     attr,
 )
 from .. import (
     pycompat,
     util,
 )
 
 shortcuts = {
     'cl': [('cmd', ['changelog']), ('rev', None)],
     'sl': [('cmd', ['shortlog']), ('rev', None)],
     'cs': [('cmd', ['changeset']), ('node', None)],
     'f': [('cmd', ['file']), ('filenode', None)],
     'fl': [('cmd', ['filelog']), ('filenode', None)],
     'fd': [('cmd', ['filediff']), ('node', None)],
     'fa': [('cmd', ['annotate']), ('filenode', None)],
     'mf': [('cmd', ['manifest']), ('manifest', None)],
     'ca': [('cmd', ['archive']), ('node', None)],
     'tags': [('cmd', ['tags'])],
     'tip': [('cmd', ['changeset']), ('node', ['tip'])],
     'static': [('cmd', ['static']), ('file', None)]
 }
 
 def normalize(form):
     # first expand the shortcuts
     for k in shortcuts:
         if k in form:
             for name, value in shortcuts[k]:
                 if value is None:
                     value = form[k]
                 form[name] = value
             del form[k]
     # And strip the values
     bytesform = {}
     for k, v in form.iteritems():
         bytesform[pycompat.bytesurl(k)] = [
             pycompat.bytesurl(i.strip()) for i in v]
     return bytesform
 
 @attr.s(frozen=True)
 class parsedrequest(object):
     """Represents a parsed WSGI request / static HTTP request parameters."""
 
     # Full URL for this request.
     url = attr.ib()
     # URL without any path components. Just <proto>://<host><port>.
     baseurl = attr.ib()
     # Advertised URL. Like ``url`` and ``baseurl`` but uses SERVER_NAME instead
     # of HTTP: Host header for hostname. This is likely what clients used.
     advertisedurl = attr.ib()
     advertisedbaseurl = attr.ib()
     # WSGI application path.
     apppath = attr.ib()
     # List of path parts to be used for dispatch.
     dispatchparts = attr.ib()
     # URL path component (no query string) used for dispatch.
     dispatchpath = attr.ib()
     # Raw query string (part after "?" in URL).
     querystring = attr.ib()
+    # List of 2-tuples of query string arguments.
+    querystringlist = attr.ib()
+    # Dict of query string arguments. Values are lists with at least 1 item.
+    querystringdict = attr.ib()
 
 def parserequestfromenv(env):
     """Parse URL components from environment variables.
 
     WSGI defines request attributes via environment variables. This function
     parses the environment variables into a data structure.
     """
     # PEP-0333 defines the WSGI spec and is a useful reference for this code.
 
     # We first validate that the incoming object conforms with the WSGI spec.
     # We only want to be dealing with spec-conforming WSGI implementations.
     # TODO enable this once we fix internal violations.
     #wsgiref.validate.check_environ(env)
 
     # PEP-0333 states that environment keys and values are native strings
     # (bytes on Python 2 and str on Python 3). The code points for the Unicode
     # strings on Python 3 must be between \00000-\000FF. We deal with bytes
     # in Mercurial, so mass convert string keys and values to bytes.
     if pycompat.ispy3:
         env = {k.encode('latin-1'): v for k, v in env.iteritems()}
         env = {k: v.encode('latin-1') if isinstance(v, str) else v
                for k, v in env.iteritems()}
 
     # https://www.python.org/dev/peps/pep-0333/#environ-variables defines
     # the environment variables.
     # https://www.python.org/dev/peps/pep-0333/#url-reconstruction defines
     # how URLs are reconstructed.
     fullurl = env['wsgi.url_scheme'] + '://'
     advertisedfullurl = fullurl
 
     def addport(s):
         if env['wsgi.url_scheme'] == 'https':
             if env['SERVER_PORT'] != '443':
                 s += ':' + env['SERVER_PORT']
         else:
             if env['SERVER_PORT'] != '80':
                 s += ':' + env['SERVER_PORT']
 
         return s
 
     if env.get('HTTP_HOST'):
         fullurl += env['HTTP_HOST']
     else:
         fullurl += env['SERVER_NAME']
         fullurl = addport(fullurl)
 
     advertisedfullurl += env['SERVER_NAME']
     advertisedfullurl = addport(advertisedfullurl)
 
     baseurl = fullurl
     advertisedbaseurl = advertisedfullurl
 
     fullurl += util.urlreq.quote(env.get('SCRIPT_NAME', ''))
     advertisedfullurl += util.urlreq.quote(env.get('SCRIPT_NAME', ''))
     fullurl += util.urlreq.quote(env.get('PATH_INFO', ''))
     advertisedfullurl += util.urlreq.quote(env.get('PATH_INFO', ''))
 
     if env.get('QUERY_STRING'):
         fullurl += '?' + env['QUERY_STRING']
         advertisedfullurl += '?' + env['QUERY_STRING']
 
     # When dispatching requests, we look at the URL components (PATH_INFO
     # and QUERY_STRING) after the application root (SCRIPT_NAME). But hgwebdir
     # has the concept of "virtual" repositories. This is defined via REPO_NAME.
     # If REPO_NAME is defined, we append it to SCRIPT_NAME to form a new app
     # root. We also exclude its path components from PATH_INFO when resolving
     # the dispatch path.
 
     apppath = env['SCRIPT_NAME']
 
     if env.get('REPO_NAME'):
         if not apppath.endswith('/'):
             apppath += '/'
 
         apppath += env.get('REPO_NAME')
 
     if 'PATH_INFO' in env:
         dispatchparts = env['PATH_INFO'].strip('/').split('/')
 
         # Strip out repo parts.
         repoparts = env.get('REPO_NAME', '').split('/')
         if dispatchparts[:len(repoparts)] == repoparts:
             dispatchparts = dispatchparts[len(repoparts):]
     else:
         dispatchparts = []
 
     dispatchpath = '/'.join(dispatchparts)
 
     querystring = env.get('QUERY_STRING', '')
 
+    # We store as a list so we have ordering information. We also store as
+    # a dict to facilitate fast lookup.
+    querystringlist = util.urlreq.parseqsl(querystring, keep_blank_values=True)
+
+    querystringdict = {}
+    for k, v in querystringlist:
+        if k in querystringdict:
+            querystringdict[k].append(v)
+        else:
+            querystringdict[k] = [v]
+
     return parsedrequest(url=fullurl, baseurl=baseurl,
                          advertisedurl=advertisedfullurl,
                          advertisedbaseurl=advertisedbaseurl,
                          apppath=apppath,
                          dispatchparts=dispatchparts, dispatchpath=dispatchpath,
-                         querystring=querystring)
+                         querystring=querystring,
+                         querystringlist=querystringlist,
+                         querystringdict=querystringdict)
 
 class wsgirequest(object):
     """Higher-level API for a WSGI request.
 
     WSGI applications are invoked with 2 arguments. They are used to
     instantiate instances of this class, which provides higher-level APIs
     for obtaining request parameters, writing HTTP output, etc.
     """
     def __init__(self, wsgienv, start_response):
         version = wsgienv[r'wsgi.version']
         if (version < (1, 0)) or (version >= (2, 0)):
             raise RuntimeError("Unknown and unsupported WSGI version %d.%d"
                                % version)
         self.inp = wsgienv[r'wsgi.input']
         self.err = wsgienv[r'wsgi.errors']
         self.threaded = wsgienv[r'wsgi.multithread']
         self.multiprocess = wsgienv[r'wsgi.multiprocess']
         self.run_once = wsgienv[r'wsgi.run_once']
         self.env = wsgienv
         self.form = normalize(cgi.parse(self.inp,
                                         self.env,
                                         keep_blank_values=1))
         self._start_response = start_response
         self.server_write = None
         self.headers = []
 
     def __iter__(self):
         return iter([])
 
     def read(self, count=-1):
         return self.inp.read(count)
 
     def drain(self):
         '''need to read all data from request, httplib is half-duplex'''
         length = int(self.env.get('CONTENT_LENGTH') or 0)
         for s in util.filechunkiter(self.inp, limit=length):
             pass
 
     def respond(self, status, type, filename=None, body=None):
         if not isinstance(type, str):
             type = pycompat.sysstr(type)
         if self._start_response is not None:
             self.headers.append((r'Content-Type', type))
             if filename:
                 filename = (filename.rpartition('/')[-1]
                             .replace('\\', '\\\\').replace('"', '\\"'))
                 self.headers.append(('Content-Disposition',
                                      'inline; filename="%s"' % filename))
             if body is not None:
                 self.headers.append((r'Content-Length', str(len(body))))
 
             for k, v in self.headers:
                 if not isinstance(v, str):
                     raise TypeError('header value must be string: %r' % (v,))
 
             if isinstance(status, ErrorResponse):
                 self.headers.extend(status.headers)
                 if status.code == HTTP_NOT_MODIFIED:
                     # RFC 2616 Section 10.3.5: 304 Not Modified has cases where
                     # it MUST NOT include any headers other than these and no
                     # body
                     self.headers = [(k, v) for (k, v) in self.headers if
                                     k in ('Date', 'ETag', 'Expires',
                                           'Cache-Control', 'Vary')]
                 status = statusmessage(status.code, pycompat.bytestr(status))
             elif status == 200:
                 status = '200 Script output follows'
             elif isinstance(status, int):
                 status = statusmessage(status)
 
             self.server_write = self._start_response(
                 pycompat.sysstr(status), self.headers)
             self._start_response = None
             self.headers = []
         if body is not None:
             self.write(body)
             self.server_write = None
 
     def write(self, thing):
         if thing:
             try:
                 self.server_write(thing)
             except socket.error as inst:
                 if inst[0] != errno.ECONNRESET:
                     raise
 
     def writelines(self, lines):
         for line in lines:
             self.write(line)
 
     def flush(self):
         return None
 
     def close(self):
         return None
 
 def wsgiapplication(app_maker):
     '''For compatibility with old CGI scripts. A plain hgweb() or hgwebdir()
     can and should now be used as a WSGI application.'''
     application = app_maker()
     def run_wsgi(env, respond):
         return application(env, respond)
     return run_wsgi

mercurial/urllibcompat.py

@@ -1,190 +1,192 b''
 # urllibcompat.py - adapters to ease using urllib2 on Py2 and urllib on Py3
 #
 # Copyright 2017 Google, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 from . import pycompat
 
 _sysstr = pycompat.sysstr
 
 class _pycompatstub(object):
     def __init__(self):
         self._aliases = {}
 
     def _registeraliases(self, origin, items):
         """Add items that will be populated at the first access"""
         items = map(_sysstr, items)
         self._aliases.update(
             (item.replace(_sysstr('_'), _sysstr('')).lower(), (origin, item))
             for item in items)
 
     def _registeralias(self, origin, attr, name):
         """Alias ``origin``.``attr`` as ``name``"""
         self._aliases[_sysstr(name)] = (origin, _sysstr(attr))
 
     def __getattr__(self, name):
         try:
             origin, item = self._aliases[name]
         except KeyError:
             raise AttributeError(name)
         self.__dict__[name] = obj = getattr(origin, item)
         return obj
 
 httpserver = _pycompatstub()
 urlreq = _pycompatstub()
 urlerr = _pycompatstub()
 
 if pycompat.ispy3:
     import urllib.parse
     urlreq._registeraliases(urllib.parse, (
         "splitattr",
         "splitpasswd",
         "splitport",
         "splituser",
         "urlparse",
         "urlunparse",
     ))
     urlreq._registeralias(urllib.parse, "parse_qs", "parseqs")
+    urlreq._registeralias(urllib.parse, "parse_qsl", "parseqsl")
     urlreq._registeralias(urllib.parse, "unquote_to_bytes", "unquote")
     import urllib.request
     urlreq._registeraliases(urllib.request, (
         "AbstractHTTPHandler",
         "BaseHandler",
         "build_opener",
         "FileHandler",
         "FTPHandler",
         "ftpwrapper",
         "HTTPHandler",
         "HTTPSHandler",
         "install_opener",
         "pathname2url",
         "HTTPBasicAuthHandler",
         "HTTPDigestAuthHandler",
         "HTTPPasswordMgrWithDefaultRealm",
         "ProxyHandler",
         "Request",
         "url2pathname",
         "urlopen",
     ))
     import urllib.response
     urlreq._registeraliases(urllib.response, (
         "addclosehook",
         "addinfourl",
     ))
     import urllib.error
     urlerr._registeraliases(urllib.error, (
         "HTTPError",
         "URLError",
     ))
     import http.server
     httpserver._registeraliases(http.server, (
         "HTTPServer",
         "BaseHTTPRequestHandler",
         "SimpleHTTPRequestHandler",
         "CGIHTTPRequestHandler",
     ))
 
     # urllib.parse.quote() accepts both str and bytes, decodes bytes
     # (if necessary), and returns str. This is wonky. We provide a custom
     # implementation that only accepts bytes and emits bytes.
     def quote(s, safe=r'/'):
         s = urllib.parse.quote_from_bytes(s, safe=safe)
         return s.encode('ascii', 'strict')
 
     # urllib.parse.urlencode() returns str. We use this function to make
     # sure we return bytes.
     def urlencode(query, doseq=False):
             s = urllib.parse.urlencode(query, doseq=doseq)
             return s.encode('ascii')
 
     urlreq.quote = quote
     urlreq.urlencode = urlencode
 
     def getfullurl(req):
         return req.full_url
 
     def gethost(req):
         return req.host
 
     def getselector(req):
         return req.selector
 
     def getdata(req):
         return req.data
 
     def hasdata(req):
         return req.data is not None
 else:
     import BaseHTTPServer
     import CGIHTTPServer
     import SimpleHTTPServer
     import urllib2
     import urllib
     import urlparse
     urlreq._registeraliases(urllib, (
         "addclosehook",
         "addinfourl",
         "ftpwrapper",
         "pathname2url",
         "quote",
         "splitattr",
         "splitpasswd",
         "splitport",
         "splituser",
         "unquote",
         "url2pathname",
         "urlencode",
     ))
     urlreq._registeraliases(urllib2, (
         "AbstractHTTPHandler",
         "BaseHandler",
         "build_opener",
         "FileHandler",
         "FTPHandler",
         "HTTPBasicAuthHandler",
         "HTTPDigestAuthHandler",
         "HTTPHandler",
         "HTTPPasswordMgrWithDefaultRealm",
         "HTTPSHandler",
         "install_opener",
         "ProxyHandler",
         "Request",
         "urlopen",
     ))
     urlreq._registeraliases(urlparse, (
         "urlparse",
         "urlunparse",
     ))
     urlreq._registeralias(urlparse, "parse_qs", "parseqs")
+    urlreq._registeralias(urlparse, "parse_qsl", "parseqsl")
     urlerr._registeraliases(urllib2, (
         "HTTPError",
         "URLError",
     ))
     httpserver._registeraliases(BaseHTTPServer, (
         "HTTPServer",
         "BaseHTTPRequestHandler",
     ))
     httpserver._registeraliases(SimpleHTTPServer, (
         "SimpleHTTPRequestHandler",
     ))
     httpserver._registeraliases(CGIHTTPServer, (
         "CGIHTTPRequestHandler",
     ))
 
     def gethost(req):
         return req.get_host()
 
     def getselector(req):
         return req.get_selector()
 
     def getfullurl(req):
         return req.get_full_url()
 
     def getdata(req):
         return req.get_data()
 
     def hasdata(req):
         return req.has_data()