hgweb: refactor fake file object proxy for archiving

Python's zip file writer operates on a file object. When doing work,
it periodically calls write(), flush(), and tell() on that object.

In WSGI contexts, the start_response function returns a write()
function. That's a function to write data, not a full file object.
So, when the archival code was first introduced in 2b03c6733efa in
2006, someone invented a proxy "tellable" type that wrapped a file
object like object and kept track of write count so it could
implement tell() and satisfy zipfile's needs.

When our archival code runs, it attempts to tell() the destination
and if that fails, converts it to a "tellable" instance. Our WSGI
application passes the "wsgirequest" instance to the archival
function. It fails the tell() test and is converted to a "tellable."
It's worth noting that "wsgirequest" implements flush(), so
"tellable" doesn't.

This hackery all seems very specific to the WSGI code. So this commit
moves the "tellable" type and the conversion of the destination file
object into the WSGI code. There's a chance some other caller may be
passing a file object like object that doesn't implement tell(). But
I doubt it.

As part of the refactor, our new type implements flush() and doesn't
implement __getattr__. Given the intended limited use of this type,
I want things to fail fast if there is an attempt to access attributes
because I think it is important to document which attributes are being
used for what purposes.

Differential Revision: https://phab.mercurial-scm.org/D2791

mercurial/archival.py

@@ -1,365 +1,342
 # archival.py - revision archival for mercurial
 #
 # Copyright 2006 Vadim Gelfer <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import gzip
 import os
 import struct
 import tarfile
 import time
 import zipfile
 import zlib
 
 from .i18n import _
 
 from . import (
     error,
     formatter,
     match as matchmod,
     pycompat,
     scmutil,
     util,
     vfs as vfsmod,
 )
 stringio = util.stringio
 
 # from unzip source code:
 _UNX_IFREG = 0x8000
 _UNX_IFLNK = 0xa000
 
 def tidyprefix(dest, kind, prefix):
     '''choose prefix to use for names in archive.  make sure prefix is
     safe for consumers.'''
 
     if prefix:
         prefix = util.normpath(prefix)
     else:
         if not isinstance(dest, bytes):
             raise ValueError('dest must be string if no prefix')
         prefix = os.path.basename(dest)
         lower = prefix.lower()
         for sfx in exts.get(kind, []):
             if lower.endswith(sfx):
                 prefix = prefix[:-len(sfx)]
                 break
     lpfx = os.path.normpath(util.localpath(prefix))
     prefix = util.pconvert(lpfx)
     if not prefix.endswith('/'):
         prefix += '/'
     # Drop the leading '.' path component if present, so Windows can read the
     # zip files (issue4634)
     if prefix.startswith('./'):
         prefix = prefix[2:]
     if prefix.startswith('../') or os.path.isabs(lpfx) or '/../' in prefix:
         raise error.Abort(_('archive prefix contains illegal components'))
     return prefix
 
 exts = {
     'tar': ['.tar'],
     'tbz2': ['.tbz2', '.tar.bz2'],
     'tgz': ['.tgz', '.tar.gz'],
     'zip': ['.zip'],
     }
 
 def guesskind(dest):
     for kind, extensions in exts.iteritems():
         if any(dest.endswith(ext) for ext in extensions):
             return kind
     return None
 
 def _rootctx(repo):
     # repo[0] may be hidden
     for rev in repo:
         return repo[rev]
     return repo['null']
 
 # {tags} on ctx includes local tags and 'tip', with no current way to limit
 # that to global tags.  Therefore, use {latesttag} as a substitute when
 # the distance is 0, since that will be the list of global tags on ctx.
 _defaultmetatemplate = br'''
 repo: {root}
 node: {ifcontains(rev, revset("wdir()"), "{p1node}{dirty}", "{node}")}
 branch: {branch|utf8}
 {ifeq(latesttagdistance, 0, join(latesttag % "tag: {tag}", "\n"),
       separate("\n",
                join(latesttag % "latesttag: {tag}", "\n"),
                "latesttagdistance: {latesttagdistance}",
                "changessincelatesttag: {changessincelatesttag}"))}
 '''[1:]  # drop leading '\n'
 
 def buildmetadata(ctx):
     '''build content of .hg_archival.txt'''
     repo = ctx.repo()
 
     opts = {
         'template': repo.ui.config('experimental', 'archivemetatemplate',
                                    _defaultmetatemplate)
     }
 
     out = util.stringio()
 
     fm = formatter.formatter(repo.ui, out, 'archive', opts)
     fm.startitem()
     fm.context(ctx=ctx)
     fm.data(root=_rootctx(repo).hex())
 
     if ctx.rev() is None:
         dirty = ''
         if ctx.dirty(missing=True):
             dirty = '+'
         fm.data(dirty=dirty)
     fm.end()
 
     return out.getvalue()
 
 class tarit(object):
     '''write archive to tar file or stream.  can write uncompressed,
     or compress with gzip or bzip2.'''
 
     class GzipFileWithTime(gzip.GzipFile):
 
         def __init__(self, *args, **kw):
             timestamp = None
             if r'timestamp' in kw:
                 timestamp = kw.pop(r'timestamp')
             if timestamp is None:
                 self.timestamp = time.time()
             else:
                 self.timestamp = timestamp
             gzip.GzipFile.__init__(self, *args, **kw)
 
         def _write_gzip_header(self):
             self.fileobj.write('\037\213')             # magic header
             self.fileobj.write('\010')                 # compression method
             fname = self.name
             if fname and fname.endswith('.gz'):
                 fname = fname[:-3]
             flags = 0
             if fname:
                 flags = gzip.FNAME
             self.fileobj.write(pycompat.bytechr(flags))
             gzip.write32u(self.fileobj, int(self.timestamp))
             self.fileobj.write('\002')
             self.fileobj.write('\377')
             if fname:
                 self.fileobj.write(fname + '\000')
 
     def __init__(self, dest, mtime, kind=''):
         self.mtime = mtime
         self.fileobj = None
 
         def taropen(mode, name='', fileobj=None):
             if kind == 'gz':
                 mode = mode[0:1]
                 if not fileobj:
                     fileobj = open(name, mode + 'b')
                 gzfileobj = self.GzipFileWithTime(name,
                                                   pycompat.sysstr(mode + 'b'),
                                                   zlib.Z_BEST_COMPRESSION,
                                                   fileobj, timestamp=mtime)
                 self.fileobj = gzfileobj
                 return tarfile.TarFile.taropen(
                     name, pycompat.sysstr(mode), gzfileobj)
             else:
                 return tarfile.open(
                     name, pycompat.sysstr(mode + kind), fileobj)
 
         if isinstance(dest, bytes):
             self.z = taropen('w:', name=dest)
         else:
             self.z = taropen('w|', fileobj=dest)
 
     def addfile(self, name, mode, islink, data):
         name = pycompat.fsdecode(name)
         i = tarfile.TarInfo(name)
         i.mtime = self.mtime
         i.size = len(data)
         if islink:
             i.type = tarfile.SYMTYPE
             i.mode = 0o777
             i.linkname = pycompat.fsdecode(data)
             data = None
             i.size = 0
         else:
             i.mode = mode
             data = stringio(data)
         self.z.addfile(i, data)
 
     def done(self):
         self.z.close()
         if self.fileobj:
             self.fileobj.close()
 
-class tellable(object):
-    '''provide tell method for zipfile.ZipFile when writing to http
-    response file object.'''
-
-    def __init__(self, fp):
-        self.fp = fp
-        self.offset = 0
-
-    def __getattr__(self, key):
-        return getattr(self.fp, key)
-
-    def write(self, s):
-        self.fp.write(s)
-        self.offset += len(s)
-
-    def tell(self):
-        return self.offset
-
 class zipit(object):
     '''write archive to zip file or stream.  can write uncompressed,
     or compressed with deflate.'''
 
     def __init__(self, dest, mtime, compress=True):
-        if not isinstance(dest, bytes):
-            try:
-                dest.tell()
-            except (AttributeError, IOError):
-                dest = tellable(dest)
         self.z = zipfile.ZipFile(pycompat.fsdecode(dest), r'w',
                                  compress and zipfile.ZIP_DEFLATED or
                                  zipfile.ZIP_STORED)
 
         # Python's zipfile module emits deprecation warnings if we try
         # to store files with a date before 1980.
         epoch = 315532800 # calendar.timegm((1980, 1, 1, 0, 0, 0, 1, 1, 0))
         if mtime < epoch:
             mtime = epoch
 
         self.mtime = mtime
         self.date_time = time.gmtime(mtime)[:6]
 
     def addfile(self, name, mode, islink, data):
         i = zipfile.ZipInfo(pycompat.fsdecode(name), self.date_time)
         i.compress_type = self.z.compression
         # unzip will not honor unix file modes unless file creator is
         # set to unix (id 3).
         i.create_system = 3
         ftype = _UNX_IFREG
         if islink:
             mode = 0o777
             ftype = _UNX_IFLNK
         i.external_attr = (mode | ftype) << 16
         # add "extended-timestamp" extra block, because zip archives
         # without this will be extracted with unexpected timestamp,
         # if TZ is not configured as GMT
         i.extra += struct.pack('<hhBl',
                                0x5455,     # block type: "extended-timestamp"
                                1 + 4,      # size of this block
                                1,          # "modification time is present"
                                int(self.mtime)) # last modification (UTC)
         self.z.writestr(i, data)
 
     def done(self):
         self.z.close()
 
 class fileit(object):
     '''write archive as files in directory.'''
 
     def __init__(self, name, mtime):
         self.basedir = name
         self.opener = vfsmod.vfs(self.basedir)
         self.mtime = mtime
 
     def addfile(self, name, mode, islink, data):
         if islink:
             self.opener.symlink(data, name)
             return
         f = self.opener(name, "w", atomictemp=False)
         f.write(data)
         f.close()
         destfile = os.path.join(self.basedir, name)
         os.chmod(destfile, mode)
         if self.mtime is not None:
             os.utime(destfile, (self.mtime, self.mtime))
 
     def done(self):
         pass
 
 archivers = {
     'files': fileit,
     'tar': tarit,
     'tbz2': lambda name, mtime: tarit(name, mtime, 'bz2'),
     'tgz': lambda name, mtime: tarit(name, mtime, 'gz'),
     'uzip': lambda name, mtime: zipit(name, mtime, False),
     'zip': zipit,
     }
 
 def archive(repo, dest, node, kind, decode=True, matchfn=None,
             prefix='', mtime=None, subrepos=False):
     '''create archive of repo as it was at node.
 
     dest can be name of directory, name of archive file, or file
     object to write archive to.
 
     kind is type of archive to create.
 
     decode tells whether to put files through decode filters from
     hgrc.
 
     matchfn is function to filter names of files to write to archive.
 
     prefix is name of path to put before every archive member.
 
     mtime is the modified time, in seconds, or None to use the changeset time.
 
     subrepos tells whether to include subrepos.
     '''
 
     if kind == 'files':
         if prefix:
             raise error.Abort(_('cannot give prefix when archiving to files'))
     else:
         prefix = tidyprefix(dest, kind, prefix)
 
     def write(name, mode, islink, getdata):
         data = getdata()
         if decode:
             data = repo.wwritedata(name, data)
         archiver.addfile(prefix + name, mode, islink, data)
 
     if kind not in archivers:
         raise error.Abort(_("unknown archive type '%s'") % kind)
 
     ctx = repo[node]
     archiver = archivers[kind](dest, mtime or ctx.date()[0])
 
     if repo.ui.configbool("ui", "archivemeta"):
         name = '.hg_archival.txt'
         if not matchfn or matchfn(name):
             write(name, 0o644, False, lambda: buildmetadata(ctx))
 
     if matchfn:
         files = [f for f in ctx.manifest().keys() if matchfn(f)]
     else:
         files = ctx.manifest().keys()
     total = len(files)
     if total:
         files.sort()
         scmutil.fileprefetchhooks(repo, ctx, files)
         repo.ui.progress(_('archiving'), 0, unit=_('files'), total=total)
         for i, f in enumerate(files):
             ff = ctx.flags(f)
             write(f, 'x' in ff and 0o755 or 0o644, 'l' in ff, ctx[f].data)
             repo.ui.progress(_('archiving'), i + 1, item=f,
                              unit=_('files'), total=total)
         repo.ui.progress(_('archiving'), None)
 
     if subrepos:
         for subpath in sorted(ctx.substate):
             sub = ctx.workingsub(subpath)
             submatch = matchmod.subdirmatcher(subpath, matchfn)
             total += sub.archive(archiver, prefix, submatch, decode)
 
     if total == 0:
         raise error.Abort(_('no files match the archive pattern'))
 
     archiver.done()
     return total

mercurial/hgweb/request.py

@@ -1,550 +1,581
 # hgweb/request.py - An http request from either CGI or the standalone server.
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005, 2006 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import errno
 import socket
 import wsgiref.headers as wsgiheaders
 #import wsgiref.validate
 
 from .common import (
     ErrorResponse,
     HTTP_NOT_MODIFIED,
     statusmessage,
 )
 
 from ..thirdparty import (
     attr,
 )
 from .. import (
     error,
     pycompat,
     util,
 )
 
 class multidict(object):
     """A dict like object that can store multiple values for a key.
 
     Used to store parsed request parameters.
 
     This is inspired by WebOb's class of the same name.
     """
     def __init__(self):
         # Stores (key, value) 2-tuples. This isn't the most efficient. But we
         # don't rely on parameters that much, so it shouldn't be a perf issue.
         # we can always add dict for fast lookups.
         self._items = []
 
     def __getitem__(self, key):
         """Returns the last set value for a key."""
         for k, v in reversed(self._items):
             if k == key:
                 return v
 
         raise KeyError(key)
 
     def __setitem__(self, key, value):
         """Replace a values for a key with a new value."""
         try:
             del self[key]
         except KeyError:
             pass
 
         self._items.append((key, value))
 
     def __delitem__(self, key):
         """Delete all values for a key."""
         oldlen = len(self._items)
 
         self._items[:] = [(k, v) for k, v in self._items if k != key]
 
         if oldlen == len(self._items):
             raise KeyError(key)
 
     def __contains__(self, key):
         return any(k == key for k, v in self._items)
 
     def __len__(self):
         return len(self._items)
 
     def get(self, key, default=None):
         try:
             return self.__getitem__(key)
         except KeyError:
             return default
 
     def add(self, key, value):
         """Add a new value for a key. Does not replace existing values."""
         self._items.append((key, value))
 
     def getall(self, key):
         """Obtains all values for a key."""
         return [v for k, v in self._items if k == key]
 
     def getone(self, key):
         """Obtain a single value for a key.
 
         Raises KeyError if key not defined or it has multiple values set.
         """
         vals = self.getall(key)
 
         if not vals:
             raise KeyError(key)
 
         if len(vals) > 1:
             raise KeyError('multiple values for %r' % key)
 
         return vals[0]
 
     def asdictoflists(self):
         d = {}
         for k, v in self._items:
             if k in d:
                 d[k].append(v)
             else:
                 d[k] = [v]
 
         return d
 
 @attr.s(frozen=True)
 class parsedrequest(object):
     """Represents a parsed WSGI request.
 
     Contains both parsed parameters as well as a handle on the input stream.
     """
 
     # Request method.
     method = attr.ib()
     # Full URL for this request.
     url = attr.ib()
     # URL without any path components. Just <proto>://<host><port>.
     baseurl = attr.ib()
     # Advertised URL. Like ``url`` and ``baseurl`` but uses SERVER_NAME instead
     # of HTTP: Host header for hostname. This is likely what clients used.
     advertisedurl = attr.ib()
     advertisedbaseurl = attr.ib()
     # URL scheme (part before ``://``). e.g. ``http`` or ``https``.
     urlscheme = attr.ib()
     # Value of REMOTE_USER, if set, or None.
     remoteuser = attr.ib()
     # Value of REMOTE_HOST, if set, or None.
     remotehost = attr.ib()
     # WSGI application path.
     apppath = attr.ib()
     # List of path parts to be used for dispatch.
     dispatchparts = attr.ib()
     # URL path component (no query string) used for dispatch.
     dispatchpath = attr.ib()
     # Whether there is a path component to this request. This can be true
     # when ``dispatchpath`` is empty due to REPO_NAME muckery.
     havepathinfo = attr.ib()
     # The name of the repository being accessed.
     reponame = attr.ib()
     # Raw query string (part after "?" in URL).
     querystring = attr.ib()
     # multidict of query string parameters.
     qsparams = attr.ib()
     # wsgiref.headers.Headers instance. Operates like a dict with case
     # insensitive keys.
     headers = attr.ib()
     # Request body input stream.
     bodyfh = attr.ib()
 
 def parserequestfromenv(env, bodyfh):
     """Parse URL components from environment variables.
 
     WSGI defines request attributes via environment variables. This function
     parses the environment variables into a data structure.
     """
     # PEP-0333 defines the WSGI spec and is a useful reference for this code.
 
     # We first validate that the incoming object conforms with the WSGI spec.
     # We only want to be dealing with spec-conforming WSGI implementations.
     # TODO enable this once we fix internal violations.
     #wsgiref.validate.check_environ(env)
 
     # PEP-0333 states that environment keys and values are native strings
     # (bytes on Python 2 and str on Python 3). The code points for the Unicode
     # strings on Python 3 must be between \00000-\000FF. We deal with bytes
     # in Mercurial, so mass convert string keys and values to bytes.
     if pycompat.ispy3:
         env = {k.encode('latin-1'): v for k, v in env.iteritems()}
         env = {k: v.encode('latin-1') if isinstance(v, str) else v
                for k, v in env.iteritems()}
 
     # https://www.python.org/dev/peps/pep-0333/#environ-variables defines
     # the environment variables.
     # https://www.python.org/dev/peps/pep-0333/#url-reconstruction defines
     # how URLs are reconstructed.
     fullurl = env['wsgi.url_scheme'] + '://'
     advertisedfullurl = fullurl
 
     def addport(s):
         if env['wsgi.url_scheme'] == 'https':
             if env['SERVER_PORT'] != '443':
                 s += ':' + env['SERVER_PORT']
         else:
             if env['SERVER_PORT'] != '80':
                 s += ':' + env['SERVER_PORT']
 
         return s
 
     if env.get('HTTP_HOST'):
         fullurl += env['HTTP_HOST']
     else:
         fullurl += env['SERVER_NAME']
         fullurl = addport(fullurl)
 
     advertisedfullurl += env['SERVER_NAME']
     advertisedfullurl = addport(advertisedfullurl)
 
     baseurl = fullurl
     advertisedbaseurl = advertisedfullurl
 
     fullurl += util.urlreq.quote(env.get('SCRIPT_NAME', ''))
     advertisedfullurl += util.urlreq.quote(env.get('SCRIPT_NAME', ''))
     fullurl += util.urlreq.quote(env.get('PATH_INFO', ''))
     advertisedfullurl += util.urlreq.quote(env.get('PATH_INFO', ''))
 
     if env.get('QUERY_STRING'):
         fullurl += '?' + env['QUERY_STRING']
         advertisedfullurl += '?' + env['QUERY_STRING']
 
     # When dispatching requests, we look at the URL components (PATH_INFO
     # and QUERY_STRING) after the application root (SCRIPT_NAME). But hgwebdir
     # has the concept of "virtual" repositories. This is defined via REPO_NAME.
     # If REPO_NAME is defined, we append it to SCRIPT_NAME to form a new app
     # root. We also exclude its path components from PATH_INFO when resolving
     # the dispatch path.
 
     apppath = env['SCRIPT_NAME']
 
     if env.get('REPO_NAME'):
         if not apppath.endswith('/'):
             apppath += '/'
 
         apppath += env.get('REPO_NAME')
 
     if 'PATH_INFO' in env:
         dispatchparts = env['PATH_INFO'].strip('/').split('/')
 
         # Strip out repo parts.
         repoparts = env.get('REPO_NAME', '').split('/')
         if dispatchparts[:len(repoparts)] == repoparts:
             dispatchparts = dispatchparts[len(repoparts):]
     else:
         dispatchparts = []
 
     dispatchpath = '/'.join(dispatchparts)
 
     querystring = env.get('QUERY_STRING', '')
 
     # We store as a list so we have ordering information. We also store as
     # a dict to facilitate fast lookup.
     qsparams = multidict()
     for k, v in util.urlreq.parseqsl(querystring, keep_blank_values=True):
         qsparams.add(k, v)
 
     # HTTP_* keys contain HTTP request headers. The Headers structure should
     # perform case normalization for us. We just rewrite underscore to dash
     # so keys match what likely went over the wire.
     headers = []
     for k, v in env.iteritems():
         if k.startswith('HTTP_'):
             headers.append((k[len('HTTP_'):].replace('_', '-'), v))
 
     headers = wsgiheaders.Headers(headers)
 
     # This is kind of a lie because the HTTP header wasn't explicitly
     # sent. But for all intents and purposes it should be OK to lie about
     # this, since a consumer will either either value to determine how many
     # bytes are available to read.
     if 'CONTENT_LENGTH' in env and 'HTTP_CONTENT_LENGTH' not in env:
         headers['Content-Length'] = env['CONTENT_LENGTH']
 
     # TODO do this once we remove wsgirequest.inp, otherwise we could have
     # multiple readers from the underlying input stream.
     #bodyfh = env['wsgi.input']
     #if 'Content-Length' in headers:
     #    bodyfh = util.cappedreader(bodyfh, int(headers['Content-Length']))
 
     return parsedrequest(method=env['REQUEST_METHOD'],
                          url=fullurl, baseurl=baseurl,
                          advertisedurl=advertisedfullurl,
                          advertisedbaseurl=advertisedbaseurl,
                          urlscheme=env['wsgi.url_scheme'],
                          remoteuser=env.get('REMOTE_USER'),
                          remotehost=env.get('REMOTE_HOST'),
                          apppath=apppath,
                          dispatchparts=dispatchparts, dispatchpath=dispatchpath,
                          havepathinfo='PATH_INFO' in env,
                          reponame=env.get('REPO_NAME'),
                          querystring=querystring,
                          qsparams=qsparams,
                          headers=headers,
                          bodyfh=bodyfh)
 
+class offsettrackingwriter(object):
+    """A file object like object that is append only and tracks write count.
+
+    Instances are bound to a callable. This callable is called with data
+    whenever a ``write()`` is attempted.
+
+    Instances track the amount of written data so they can answer ``tell()``
+    requests.
+
+    The intent of this class is to wrap the ``write()`` function returned by
+    a WSGI ``start_response()`` function. Since ``write()`` is a callable and
+    not a file object, it doesn't implement other file object methods.
+    """
+    def __init__(self, writefn):
+        self._write = writefn
+        self._offset = 0
+
+    def write(self, s):
+        res = self._write(s)
+        # Some Python objects don't report the number of bytes written.
+        if res is None:
+            self._offset += len(s)
+        else:
+            self._offset += res
+
+    def flush(self):
+        pass
+
+    def tell(self):
+        return self._offset
+
 class wsgiresponse(object):
     """Represents a response to a WSGI request.
 
     A response consists of a status line, headers, and a body.
 
     Consumers must populate the ``status`` and ``headers`` fields and
     make a call to a ``setbody*()`` method before the response can be
     issued.
 
     When it is time to start sending the response over the wire,
     ``sendresponse()`` is called. It handles emitting the header portion
     of the response message. It then yields chunks of body data to be
     written to the peer. Typically, the WSGI application itself calls
     and returns the value from ``sendresponse()``.
     """
 
     def __init__(self, req, startresponse):
         """Create an empty response tied to a specific request.
 
         ``req`` is a ``parsedrequest``. ``startresponse`` is the
         ``start_response`` function passed to the WSGI application.
         """
         self._req = req
         self._startresponse = startresponse
 
         self.status = None
         self.headers = wsgiheaders.Headers([])
 
         self._bodybytes = None
         self._bodygen = None
         self._started = False
 
     def setbodybytes(self, b):
         """Define the response body as static bytes."""
         if self._bodybytes is not None or self._bodygen is not None:
             raise error.ProgrammingError('cannot define body multiple times')
 
         self._bodybytes = b
         self.headers['Content-Length'] = '%d' % len(b)
 
     def setbodygen(self, gen):
         """Define the response body as a generator of bytes."""
         if self._bodybytes is not None or self._bodygen is not None:
             raise error.ProgrammingError('cannot define body multiple times')
 
         self._bodygen = gen
 
     def sendresponse(self):
         """Send the generated response to the client.
 
         Before this is called, ``status`` must be set and one of
         ``setbodybytes()`` or ``setbodygen()`` must be called.
 
         Calling this method multiple times is not allowed.
         """
         if self._started:
             raise error.ProgrammingError('sendresponse() called multiple times')
 
         self._started = True
 
         if not self.status:
             raise error.ProgrammingError('status line not defined')
 
         if self._bodybytes is None and self._bodygen is None:
             raise error.ProgrammingError('response body not defined')
 
         # Various HTTP clients (notably httplib) won't read the HTTP response
         # until the HTTP request has been sent in full. If servers (us) send a
         # response before the HTTP request has been fully sent, the connection
         # may deadlock because neither end is reading.
         #
         # We work around this by "draining" the request data before
         # sending any response in some conditions.
         drain = False
         close = False
 
         # If the client sent Expect: 100-continue, we assume it is smart enough
         # to deal with the server sending a response before reading the request.
         # (httplib doesn't do this.)
         if self._req.headers.get('Expect', '').lower() == '100-continue':
             pass
         # Only tend to request methods that have bodies. Strictly speaking,
         # we should sniff for a body. But this is fine for our existing
         # WSGI applications.
         elif self._req.method not in ('POST', 'PUT'):
             pass
         else:
             # If we don't know how much data to read, there's no guarantee
             # that we can drain the request responsibly. The WSGI
             # specification only says that servers *should* ensure the
             # input stream doesn't overrun the actual request. So there's
             # no guarantee that reading until EOF won't corrupt the stream
             # state.
             if not isinstance(self._req.bodyfh, util.cappedreader):
                 close = True
             else:
                 # We /could/ only drain certain HTTP response codes. But 200 and
                 # non-200 wire protocol responses both require draining. Since
                 # we have a capped reader in place for all situations where we
                 # drain, it is safe to read from that stream. We'll either do
                 # a drain or no-op if we're already at EOF.
                 drain = True
 
         if close:
             self.headers['Connection'] = 'Close'
 
         if drain:
             assert isinstance(self._req.bodyfh, util.cappedreader)
             while True:
                 chunk = self._req.bodyfh.read(32768)
                 if not chunk:
                     break
 
         self._startresponse(pycompat.sysstr(self.status), self.headers.items())
         if self._bodybytes:
             yield self._bodybytes
         elif self._bodygen:
             for chunk in self._bodygen:
                 yield chunk
         else:
             error.ProgrammingError('do not know how to send body')
 
 class wsgirequest(object):
     """Higher-level API for a WSGI request.
 
     WSGI applications are invoked with 2 arguments. They are used to
     instantiate instances of this class, which provides higher-level APIs
     for obtaining request parameters, writing HTTP output, etc.
     """
     def __init__(self, wsgienv, start_response):
         version = wsgienv[r'wsgi.version']
         if (version < (1, 0)) or (version >= (2, 0)):
             raise RuntimeError("Unknown and unsupported WSGI version %d.%d"
                                % version)
 
         inp = wsgienv[r'wsgi.input']
 
         if r'HTTP_CONTENT_LENGTH' in wsgienv:
             inp = util.cappedreader(inp, int(wsgienv[r'HTTP_CONTENT_LENGTH']))
         elif r'CONTENT_LENGTH' in wsgienv:
             inp = util.cappedreader(inp, int(wsgienv[r'CONTENT_LENGTH']))
 
         self.err = wsgienv[r'wsgi.errors']
         self.threaded = wsgienv[r'wsgi.multithread']
         self.multiprocess = wsgienv[r'wsgi.multiprocess']
         self.run_once = wsgienv[r'wsgi.run_once']
         self.env = wsgienv
         self.req = parserequestfromenv(wsgienv, inp)
         self.res = wsgiresponse(self.req, start_response)
         self._start_response = start_response
         self.server_write = None
         self.headers = []
 
     def respond(self, status, type, filename=None, body=None):
         if not isinstance(type, str):
             type = pycompat.sysstr(type)
         if self._start_response is not None:
             self.headers.append((r'Content-Type', type))
             if filename:
                 filename = (filename.rpartition('/')[-1]
                             .replace('\\', '\\\\').replace('"', '\\"'))
                 self.headers.append(('Content-Disposition',
                                      'inline; filename="%s"' % filename))
             if body is not None:
                 self.headers.append((r'Content-Length', str(len(body))))
 
             for k, v in self.headers:
                 if not isinstance(v, str):
                     raise TypeError('header value must be string: %r' % (v,))
 
             if isinstance(status, ErrorResponse):
                 self.headers.extend(status.headers)
                 if status.code == HTTP_NOT_MODIFIED:
                     # RFC 2616 Section 10.3.5: 304 Not Modified has cases where
                     # it MUST NOT include any headers other than these and no
                     # body
                     self.headers = [(k, v) for (k, v) in self.headers if
                                     k in ('Date', 'ETag', 'Expires',
                                           'Cache-Control', 'Vary')]
                 status = statusmessage(status.code, pycompat.bytestr(status))
             elif status == 200:
                 status = '200 Script output follows'
             elif isinstance(status, int):
                 status = statusmessage(status)
 
             # Various HTTP clients (notably httplib) won't read the HTTP
             # response until the HTTP request has been sent in full. If servers
             # (us) send a response before the HTTP request has been fully sent,
             # the connection may deadlock because neither end is reading.
             #
             # We work around this by "draining" the request data before
             # sending any response in some conditions.
             drain = False
             close = False
 
             # If the client sent Expect: 100-continue, we assume it is smart
             # enough to deal with the server sending a response before reading
             # the request. (httplib doesn't do this.)
             if self.env.get(r'HTTP_EXPECT', r'').lower() == r'100-continue':
                 pass
             # Only tend to request methods that have bodies. Strictly speaking,
             # we should sniff for a body. But this is fine for our existing
             # WSGI applications.
             elif self.env[r'REQUEST_METHOD'] not in (r'POST', r'PUT'):
                 pass
             else:
                 # If we don't know how much data to read, there's no guarantee
                 # that we can drain the request responsibly. The WSGI
                 # specification only says that servers *should* ensure the
                 # input stream doesn't overrun the actual request. So there's
                 # no guarantee that reading until EOF won't corrupt the stream
                 # state.
                 if not isinstance(self.req.bodyfh, util.cappedreader):
                     close = True
                 else:
                     # We /could/ only drain certain HTTP response codes. But 200
                     # and non-200 wire protocol responses both require draining.
                     # Since we have a capped reader in place for all situations
                     # where we drain, it is safe to read from that stream. We'll
                     # either do a drain or no-op if we're already at EOF.
                     drain = True
 
             if close:
                 self.headers.append((r'Connection', r'Close'))
 
             if drain:
                 assert isinstance(self.req.bodyfh, util.cappedreader)
                 while True:
                     chunk = self.req.bodyfh.read(32768)
                     if not chunk:
                         break
 
             self.server_write = self._start_response(
                 pycompat.sysstr(status), self.headers)
             self._start_response = None
             self.headers = []
         if body is not None:
             self.write(body)
             self.server_write = None
 
     def write(self, thing):
         if thing:
             try:
                 self.server_write(thing)
             except socket.error as inst:
                 if inst[0] != errno.ECONNRESET:
                     raise
 
     def flush(self):
         return None
 
 def wsgiapplication(app_maker):
     '''For compatibility with old CGI scripts. A plain hgweb() or hgwebdir()
     can and should now be used as a WSGI application.'''
     application = app_maker()
     def run_wsgi(env, respond):
         return application(env, respond)
     return run_wsgi

mercurial/hgweb/webcommands.py

@@ -1,1506 +1,1511
 #
 # Copyright 21 May 2005 - (c) 2005 Jake Edge <[e-mail]>
 # Copyright 2005-2007 Matt Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import copy
 import mimetypes
 import os
 import re
 
 from ..i18n import _
 from ..node import hex, nullid, short
 
 from .common import (
     ErrorResponse,
     HTTP_FORBIDDEN,
     HTTP_NOT_FOUND,
     HTTP_OK,
     get_contact,
     paritygen,
     staticfile,
 )
+from . import (
+    request as requestmod,
+)
 
 from .. import (
     archival,
     dagop,
     encoding,
     error,
     graphmod,
     pycompat,
     revset,
     revsetlang,
     scmutil,
     smartset,
     templater,
     util,
 )
 
 from . import (
     webutil,
 )
 
 __all__ = []
 commands = {}
 
 class webcommand(object):
     """Decorator used to register a web command handler.
 
     The decorator takes as its positional arguments the name/path the
     command should be accessible under.
 
     When called, functions receive as arguments a ``requestcontext``,
     ``wsgirequest``, and a templater instance for generatoring output.
     The functions should populate the ``rctx.res`` object with details
     about the HTTP response.
 
     The function can return the ``requestcontext.res`` instance to signal
     that it wants to use this object to generate the response. If an iterable
     is returned, the ``wsgirequest`` instance will be used and the returned
     content will constitute the response body.
 
     Usage:
 
     @webcommand('mycommand')
     def mycommand(web, req, tmpl):
         pass
     """
 
     def __init__(self, name):
         self.name = name
 
     def __call__(self, func):
         __all__.append(self.name)
         commands[self.name] = func
         return func
 
 @webcommand('log')
 def log(web, req, tmpl):
     """
     /log[/{revision}[/{path}]]
     --------------------------
 
     Show repository or file history.
 
     For URLs of the form ``/log/{revision}``, a list of changesets starting at
     the specified changeset identifier is shown. If ``{revision}`` is not
     defined, the default is ``tip``. This form is equivalent to the
     ``changelog`` handler.
 
     For URLs of the form ``/log/{revision}/{file}``, the history for a specific
     file will be shown. This form is equivalent to the ``filelog`` handler.
     """
 
     if req.req.qsparams.get('file'):
         return filelog(web, req, tmpl)
     else:
         return changelog(web, req, tmpl)
 
 @webcommand('rawfile')
 def rawfile(web, req, tmpl):
     guessmime = web.configbool('web', 'guessmime')
 
     path = webutil.cleanpath(web.repo, req.req.qsparams.get('file', ''))
     if not path:
         return manifest(web, req, tmpl)
 
     try:
         fctx = webutil.filectx(web.repo, req)
     except error.LookupError as inst:
         try:
             return manifest(web, req, tmpl)
         except ErrorResponse:
             raise inst
 
     path = fctx.path()
     text = fctx.data()
     mt = 'application/binary'
     if guessmime:
         mt = mimetypes.guess_type(path)[0]
         if mt is None:
             if util.binary(text):
                 mt = 'application/binary'
             else:
                 mt = 'text/plain'
     if mt.startswith('text/'):
         mt += '; charset="%s"' % encoding.encoding
 
     web.res.headers['Content-Type'] = mt
     filename = (path.rpartition('/')[-1]
                 .replace('\\', '\\\\').replace('"', '\\"'))
     web.res.headers['Content-Disposition'] = 'inline; filename="%s"' % filename
     web.res.setbodybytes(text)
     return web.res
 
 def _filerevision(web, req, tmpl, fctx):
     f = fctx.path()
     text = fctx.data()
     parity = paritygen(web.stripecount)
     ishead = fctx.filerev() in fctx.filelog().headrevs()
 
     if util.binary(text):
         mt = mimetypes.guess_type(f)[0] or 'application/octet-stream'
         text = '(binary:%s)' % mt
 
     def lines():
         for lineno, t in enumerate(text.splitlines(True)):
             yield {"line": t,
                    "lineid": "l%d" % (lineno + 1),
                    "linenumber": "% 6d" % (lineno + 1),
                    "parity": next(parity)}
 
     web.res.setbodygen(tmpl(
         'filerevision',
         file=f,
         path=webutil.up(f),
         text=lines(),
         symrev=webutil.symrevorshortnode(req, fctx),
         rename=webutil.renamelink(fctx),
         permissions=fctx.manifest().flags(f),
         ishead=int(ishead),
         **pycompat.strkwargs(webutil.commonentry(web.repo, fctx))))
 
     return web.res
 
 @webcommand('file')
 def file(web, req, tmpl):
     """
     /file/{revision}[/{path}]
     -------------------------
 
     Show information about a directory or file in the repository.
 
     Info about the ``path`` given as a URL parameter will be rendered.
 
     If ``path`` is a directory, information about the entries in that
     directory will be rendered. This form is equivalent to the ``manifest``
     handler.
 
     If ``path`` is a file, information about that file will be shown via
     the ``filerevision`` template.
 
     If ``path`` is not defined, information about the root directory will
     be rendered.
     """
     if web.req.qsparams.get('style') == 'raw':
         return rawfile(web, req, tmpl)
 
     path = webutil.cleanpath(web.repo, req.req.qsparams.get('file', ''))
     if not path:
         return manifest(web, req, tmpl)
     try:
         return _filerevision(web, req, tmpl, webutil.filectx(web.repo, req))
     except error.LookupError as inst:
         try:
             return manifest(web, req, tmpl)
         except ErrorResponse:
             raise inst
 
 def _search(web, req, tmpl):
     MODE_REVISION = 'rev'
     MODE_KEYWORD = 'keyword'
     MODE_REVSET = 'revset'
 
     def revsearch(ctx):
         yield ctx
 
     def keywordsearch(query):
         lower = encoding.lower
         qw = lower(query).split()
 
         def revgen():
             cl = web.repo.changelog
             for i in xrange(len(web.repo) - 1, 0, -100):
                 l = []
                 for j in cl.revs(max(0, i - 99), i):
                     ctx = web.repo[j]
                     l.append(ctx)
                 l.reverse()
                 for e in l:
                     yield e
 
         for ctx in revgen():
             miss = 0
             for q in qw:
                 if not (q in lower(ctx.user()) or
                         q in lower(ctx.description()) or
                         q in lower(" ".join(ctx.files()))):
                     miss = 1
                     break
             if miss:
                 continue
 
             yield ctx
 
     def revsetsearch(revs):
         for r in revs:
             yield web.repo[r]
 
     searchfuncs = {
         MODE_REVISION: (revsearch, 'exact revision search'),
         MODE_KEYWORD: (keywordsearch, 'literal keyword search'),
         MODE_REVSET: (revsetsearch, 'revset expression search'),
     }
 
     def getsearchmode(query):
         try:
             ctx = web.repo[query]
         except (error.RepoError, error.LookupError):
             # query is not an exact revision pointer, need to
             # decide if it's a revset expression or keywords
             pass
         else:
             return MODE_REVISION, ctx
 
         revdef = 'reverse(%s)' % query
         try:
             tree = revsetlang.parse(revdef)
         except error.ParseError:
             # can't parse to a revset tree
             return MODE_KEYWORD, query
 
         if revsetlang.depth(tree) <= 2:
             # no revset syntax used
             return MODE_KEYWORD, query
 
         if any((token, (value or '')[:3]) == ('string', 're:')
                for token, value, pos in revsetlang.tokenize(revdef)):
             return MODE_KEYWORD, query
 
         funcsused = revsetlang.funcsused(tree)
         if not funcsused.issubset(revset.safesymbols):
             return MODE_KEYWORD, query
 
         mfunc = revset.match(web.repo.ui, revdef, repo=web.repo)
         try:
             revs = mfunc(web.repo)
             return MODE_REVSET, revs
             # ParseError: wrongly placed tokens, wrongs arguments, etc
             # RepoLookupError: no such revision, e.g. in 'revision:'
             # Abort: bookmark/tag not exists
             # LookupError: ambiguous identifier, e.g. in '(bc)' on a large repo
         except (error.ParseError, error.RepoLookupError, error.Abort,
                 LookupError):
             return MODE_KEYWORD, query
 
     def changelist(**map):
         count = 0
 
         for ctx in searchfunc[0](funcarg):
             count += 1
             n = ctx.node()
             showtags = webutil.showtag(web.repo, tmpl, 'changelogtag', n)
             files = webutil.listfilediffs(tmpl, ctx.files(), n, web.maxfiles)
 
             yield tmpl('searchentry',
                        parity=next(parity),
                        changelogtag=showtags,
                        files=files,
                        **pycompat.strkwargs(webutil.commonentry(web.repo, ctx)))
 
             if count >= revcount:
                 break
 
     query = req.req.qsparams['rev']
     revcount = web.maxchanges
     if 'revcount' in req.req.qsparams:
         try:
             revcount = int(req.req.qsparams.get('revcount', revcount))
             revcount = max(revcount, 1)
             tmpl.defaults['sessionvars']['revcount'] = revcount
         except ValueError:
             pass
 
     lessvars = copy.copy(tmpl.defaults['sessionvars'])
     lessvars['revcount'] = max(revcount // 2, 1)
     lessvars['rev'] = query
     morevars = copy.copy(tmpl.defaults['sessionvars'])
     morevars['revcount'] = revcount * 2
     morevars['rev'] = query
 
     mode, funcarg = getsearchmode(query)
 
     if 'forcekw' in req.req.qsparams:
         showforcekw = ''
         showunforcekw = searchfuncs[mode][1]
         mode = MODE_KEYWORD
         funcarg = query
     else:
         if mode != MODE_KEYWORD:
             showforcekw = searchfuncs[MODE_KEYWORD][1]
         else:
             showforcekw = ''
         showunforcekw = ''
 
     searchfunc = searchfuncs[mode]
 
     tip = web.repo['tip']
     parity = paritygen(web.stripecount)
 
     web.res.setbodygen(tmpl(
         'search',
         query=query,
         node=tip.hex(),
         symrev='tip',
         entries=changelist,
         archives=web.archivelist('tip'),
         morevars=morevars,
         lessvars=lessvars,
         modedesc=searchfunc[1],
         showforcekw=showforcekw,
         showunforcekw=showunforcekw))
 
     return web.res
 
 @webcommand('changelog')
 def changelog(web, req, tmpl, shortlog=False):
     """
     /changelog[/{revision}]
     -----------------------
 
     Show information about multiple changesets.
 
     If the optional ``revision`` URL argument is absent, information about
     all changesets starting at ``tip`` will be rendered. If the ``revision``
     argument is present, changesets will be shown starting from the specified
     revision.
 
     If ``revision`` is absent, the ``rev`` query string argument may be
     defined. This will perform a search for changesets.
 
     The argument for ``rev`` can be a single revision, a revision set,
     or a literal keyword to search for in changeset data (equivalent to
     :hg:`log -k`).
 
     The ``revcount`` query string argument defines the maximum numbers of
     changesets to render.
 
     For non-searches, the ``changelog`` template will be rendered.
     """
 
     query = ''
     if 'node' in req.req.qsparams:
         ctx = webutil.changectx(web.repo, req)
         symrev = webutil.symrevorshortnode(req, ctx)
     elif 'rev' in req.req.qsparams:
         return _search(web, req, tmpl)
     else:
         ctx = web.repo['tip']
         symrev = 'tip'
 
     def changelist():
         revs = []
         if pos != -1:
             revs = web.repo.changelog.revs(pos, 0)
         curcount = 0
         for rev in revs:
             curcount += 1
             if curcount > revcount + 1:
                 break
 
             entry = webutil.changelistentry(web, web.repo[rev], tmpl)
             entry['parity'] = next(parity)
             yield entry
 
     if shortlog:
         revcount = web.maxshortchanges
     else:
         revcount = web.maxchanges
 
     if 'revcount' in req.req.qsparams:
         try:
             revcount = int(req.req.qsparams.get('revcount', revcount))
             revcount = max(revcount, 1)
             tmpl.defaults['sessionvars']['revcount'] = revcount
         except ValueError:
             pass
 
     lessvars = copy.copy(tmpl.defaults['sessionvars'])
     lessvars['revcount'] = max(revcount // 2, 1)
     morevars = copy.copy(tmpl.defaults['sessionvars'])
     morevars['revcount'] = revcount * 2
 
     count = len(web.repo)
     pos = ctx.rev()
     parity = paritygen(web.stripecount)
 
     changenav = webutil.revnav(web.repo).gen(pos, revcount, count)
 
     entries = list(changelist())
     latestentry = entries[:1]
     if len(entries) > revcount:
         nextentry = entries[-1:]
         entries = entries[:-1]
     else:
         nextentry = []
 
     web.res.setbodygen(tmpl(
         'shortlog' if shortlog else 'changelog',
         changenav=changenav,
         node=ctx.hex(),
         rev=pos,
         symrev=symrev,
         changesets=count,
         entries=entries,
         latestentry=latestentry,
         nextentry=nextentry,
         archives=web.archivelist('tip'),
         revcount=revcount,
         morevars=morevars,
         lessvars=lessvars,
         query=query))
 
     return web.res
 
 @webcommand('shortlog')
 def shortlog(web, req, tmpl):
     """
     /shortlog
     ---------
 
     Show basic information about a set of changesets.
 
     This accepts the same parameters as the ``changelog`` handler. The only
     difference is the ``shortlog`` template will be rendered instead of the
     ``changelog`` template.
     """
     return changelog(web, req, tmpl, shortlog=True)
 
 @webcommand('changeset')
 def changeset(web, req, tmpl):
     """
     /changeset[/{revision}]
     -----------------------
 
     Show information about a single changeset.
 
     A URL path argument is the changeset identifier to show. See ``hg help
     revisions`` for possible values. If not defined, the ``tip`` changeset
     will be shown.
 
     The ``changeset`` template is rendered. Contents of the ``changesettag``,
     ``changesetbookmark``, ``filenodelink``, ``filenolink``, and the many
     templates related to diffs may all be used to produce the output.
     """
     ctx = webutil.changectx(web.repo, req)
     web.res.setbodygen(tmpl('changeset',
                             **webutil.changesetentry(web, req, tmpl, ctx)))
     return web.res
 
 rev = webcommand('rev')(changeset)
 
 def decodepath(path):
     """Hook for mapping a path in the repository to a path in the
     working copy.
 
     Extensions (e.g., largefiles) can override this to remap files in
     the virtual file system presented by the manifest command below."""
     return path
 
 @webcommand('manifest')
 def manifest(web, req, tmpl):
     """
     /manifest[/{revision}[/{path}]]
     -------------------------------
 
     Show information about a directory.
 
     If the URL path arguments are omitted, information about the root
     directory for the ``tip`` changeset will be shown.
 
     Because this handler can only show information for directories, it
     is recommended to use the ``file`` handler instead, as it can handle both
     directories and files.
 
     The ``manifest`` template will be rendered for this handler.
     """
     if 'node' in req.req.qsparams:
         ctx = webutil.changectx(web.repo, req)
         symrev = webutil.symrevorshortnode(req, ctx)
     else:
         ctx = web.repo['tip']
         symrev = 'tip'
     path = webutil.cleanpath(web.repo, req.req.qsparams.get('file', ''))
     mf = ctx.manifest()
     node = ctx.node()
 
     files = {}
     dirs = {}
     parity = paritygen(web.stripecount)
 
     if path and path[-1:] != "/":
         path += "/"
     l = len(path)
     abspath = "/" + path
 
     for full, n in mf.iteritems():
         # the virtual path (working copy path) used for the full
         # (repository) path
         f = decodepath(full)
 
         if f[:l] != path:
             continue
         remain = f[l:]
         elements = remain.split('/')
         if len(elements) == 1:
             files[remain] = full
         else:
             h = dirs # need to retain ref to dirs (root)
             for elem in elements[0:-1]:
                 if elem not in h:
                     h[elem] = {}
                 h = h[elem]
                 if len(h) > 1:
                     break
             h[None] = None # denotes files present
 
     if mf and not files and not dirs:
         raise ErrorResponse(HTTP_NOT_FOUND, 'path not found: ' + path)
 
     def filelist(**map):
         for f in sorted(files):
             full = files[f]
 
             fctx = ctx.filectx(full)
             yield {"file": full,
                    "parity": next(parity),
                    "basename": f,
                    "date": fctx.date(),
                    "size": fctx.size(),
                    "permissions": mf.flags(full)}
 
     def dirlist(**map):
         for d in sorted(dirs):
 
             emptydirs = []
             h = dirs[d]
             while isinstance(h, dict) and len(h) == 1:
                 k, v = next(iter(h.items()))
                 if v:
                     emptydirs.append(k)
                 h = v
 
             path = "%s%s" % (abspath, d)
             yield {"parity": next(parity),
                    "path": path,
                    "emptydirs": "/".join(emptydirs),
                    "basename": d}
 
     web.res.setbodygen(tmpl(
         'manifest',
         symrev=symrev,
         path=abspath,
         up=webutil.up(abspath),
         upparity=next(parity),
         fentries=filelist,
         dentries=dirlist,
         archives=web.archivelist(hex(node)),
         **pycompat.strkwargs(webutil.commonentry(web.repo, ctx))))
 
     return web.res
 
 @webcommand('tags')
 def tags(web, req, tmpl):
     """
     /tags
     -----
 
     Show information about tags.
 
     No arguments are accepted.
 
     The ``tags`` template is rendered.
     """
     i = list(reversed(web.repo.tagslist()))
     parity = paritygen(web.stripecount)
 
     def entries(notip, latestonly, **map):
         t = i
         if notip:
             t = [(k, n) for k, n in i if k != "tip"]
         if latestonly:
             t = t[:1]
         for k, n in t:
             yield {"parity": next(parity),
                    "tag": k,
                    "date": web.repo[n].date(),
                    "node": hex(n)}
 
     web.res.setbodygen(tmpl(
         'tags',
         node=hex(web.repo.changelog.tip()),
         entries=lambda **x: entries(False, False, **x),
         entriesnotip=lambda **x: entries(True, False, **x),
         latestentry=lambda **x: entries(True, True, **x)))
 
     return web.res
 
 @webcommand('bookmarks')
 def bookmarks(web, req, tmpl):
     """
     /bookmarks
     ----------
 
     Show information about bookmarks.
 
     No arguments are accepted.
 
     The ``bookmarks`` template is rendered.
     """
     i = [b for b in web.repo._bookmarks.items() if b[1] in web.repo]
     sortkey = lambda b: (web.repo[b[1]].rev(), b[0])
     i = sorted(i, key=sortkey, reverse=True)
     parity = paritygen(web.stripecount)
 
     def entries(latestonly, **map):
         t = i
         if latestonly:
             t = i[:1]
         for k, n in t:
             yield {"parity": next(parity),
                    "bookmark": k,
                    "date": web.repo[n].date(),
                    "node": hex(n)}
 
     if i:
         latestrev = i[0][1]
     else:
         latestrev = -1
 
     web.res.setbodygen(tmpl(
         'bookmarks',
         node=hex(web.repo.changelog.tip()),
         lastchange=[{'date': web.repo[latestrev].date()}],
         entries=lambda **x: entries(latestonly=False, **x),
         latestentry=lambda **x: entries(latestonly=True, **x)))
 
     return web.res
 
 @webcommand('branches')
 def branches(web, req, tmpl):
     """
     /branches
     ---------
 
     Show information about branches.
 
     All known branches are contained in the output, even closed branches.
 
     No arguments are accepted.
 
     The ``branches`` template is rendered.
     """
     entries = webutil.branchentries(web.repo, web.stripecount)
     latestentry = webutil.branchentries(web.repo, web.stripecount, 1)
 
     web.res.setbodygen(tmpl(
         'branches',
         node=hex(web.repo.changelog.tip()),
         entries=entries,
         latestentry=latestentry))
 
     return web.res
 
 @webcommand('summary')
 def summary(web, req, tmpl):
     """
     /summary
     --------
 
     Show a summary of repository state.
 
     Information about the latest changesets, bookmarks, tags, and branches
     is captured by this handler.
 
     The ``summary`` template is rendered.
     """
     i = reversed(web.repo.tagslist())
 
     def tagentries(**map):
         parity = paritygen(web.stripecount)
         count = 0
         for k, n in i:
             if k == "tip": # skip tip
                 continue
 
             count += 1
             if count > 10: # limit to 10 tags
                 break
 
             yield tmpl("tagentry",
                        parity=next(parity),
                        tag=k,
                        node=hex(n),
                        date=web.repo[n].date())
 
     def bookmarks(**map):
         parity = paritygen(web.stripecount)
         marks = [b for b in web.repo._bookmarks.items() if b[1] in web.repo]
         sortkey = lambda b: (web.repo[b[1]].rev(), b[0])
         marks = sorted(marks, key=sortkey, reverse=True)
         for k, n in marks[:10]:  # limit to 10 bookmarks
             yield {'parity': next(parity),
                    'bookmark': k,
                    'date': web.repo[n].date(),
                    'node': hex(n)}
 
     def changelist(**map):
         parity = paritygen(web.stripecount, offset=start - end)
         l = [] # build a list in forward order for efficiency
         revs = []
         if start < end:
             revs = web.repo.changelog.revs(start, end - 1)
         for i in revs:
             ctx = web.repo[i]
 
             l.append(tmpl(
                 'shortlogentry',
                 parity=next(parity),
                 **pycompat.strkwargs(webutil.commonentry(web.repo, ctx))))
 
         for entry in reversed(l):
             yield entry
 
     tip = web.repo['tip']
     count = len(web.repo)
     start = max(0, count - web.maxchanges)
     end = min(count, start + web.maxchanges)
 
     desc = web.config("web", "description")
     if not desc:
         desc = 'unknown'
 
     web.res.setbodygen(tmpl(
         'summary',
         desc=desc,
         owner=get_contact(web.config) or 'unknown',
         lastchange=tip.date(),
         tags=tagentries,
         bookmarks=bookmarks,
         branches=webutil.branchentries(web.repo, web.stripecount, 10),
         shortlog=changelist,
         node=tip.hex(),
         symrev='tip',
         archives=web.archivelist('tip'),
         labels=web.configlist('web', 'labels')))
 
     return web.res
 
 @webcommand('filediff')
 def filediff(web, req, tmpl):
     """
     /diff/{revision}/{path}
     -----------------------
 
     Show how a file changed in a particular commit.
 
     The ``filediff`` template is rendered.
 
     This handler is registered under both the ``/diff`` and ``/filediff``
     paths. ``/diff`` is used in modern code.
     """
     fctx, ctx = None, None
     try:
         fctx = webutil.filectx(web.repo, req)
     except LookupError:
         ctx = webutil.changectx(web.repo, req)
         path = webutil.cleanpath(web.repo, req.req.qsparams['file'])
         if path not in ctx.files():
             raise
 
     if fctx is not None:
         path = fctx.path()
         ctx = fctx.changectx()
     basectx = ctx.p1()
 
     style = web.config('web', 'style')
     if 'style' in req.req.qsparams:
         style = req.req.qsparams['style']
 
     diffs = webutil.diffs(web, tmpl, ctx, basectx, [path], style)
     if fctx is not None:
         rename = webutil.renamelink(fctx)
         ctx = fctx
     else:
         rename = []
         ctx = ctx
 
     web.res.setbodygen(tmpl(
         'filediff',
         file=path,
         symrev=webutil.symrevorshortnode(req, ctx),
         rename=rename,
         diff=diffs,
         **pycompat.strkwargs(webutil.commonentry(web.repo, ctx))))
 
     return web.res
 
 diff = webcommand('diff')(filediff)
 
 @webcommand('comparison')
 def comparison(web, req, tmpl):
     """
     /comparison/{revision}/{path}
     -----------------------------
 
     Show a comparison between the old and new versions of a file from changes
     made on a particular revision.
 
     This is similar to the ``diff`` handler. However, this form features
     a split or side-by-side diff rather than a unified diff.
 
     The ``context`` query string argument can be used to control the lines of
     context in the diff.
 
     The ``filecomparison`` template is rendered.
     """
     ctx = webutil.changectx(web.repo, req)
     if 'file' not in req.req.qsparams:
         raise ErrorResponse(HTTP_NOT_FOUND, 'file not given')
     path = webutil.cleanpath(web.repo, req.req.qsparams['file'])
 
     parsecontext = lambda v: v == 'full' and -1 or int(v)
     if 'context' in req.req.qsparams:
         context = parsecontext(req.req.qsparams['context'])
     else:
         context = parsecontext(web.config('web', 'comparisoncontext', '5'))
 
     def filelines(f):
         if f.isbinary():
             mt = mimetypes.guess_type(f.path())[0]
             if not mt:
                 mt = 'application/octet-stream'
             return [_('(binary file %s, hash: %s)') % (mt, hex(f.filenode()))]
         return f.data().splitlines()
 
     fctx = None
     parent = ctx.p1()
     leftrev = parent.rev()
     leftnode = parent.node()
     rightrev = ctx.rev()
     rightnode = ctx.node()
     if path in ctx:
         fctx = ctx[path]
         rightlines = filelines(fctx)
         if path not in parent:
             leftlines = ()
         else:
             pfctx = parent[path]
             leftlines = filelines(pfctx)
     else:
         rightlines = ()
         pfctx = ctx.parents()[0][path]
         leftlines = filelines(pfctx)
 
     comparison = webutil.compare(tmpl, context, leftlines, rightlines)
     if fctx is not None:
         rename = webutil.renamelink(fctx)
         ctx = fctx
     else:
         rename = []
         ctx = ctx
 
     web.res.setbodygen(tmpl(
         'filecomparison',
         file=path,
         symrev=webutil.symrevorshortnode(req, ctx),
         rename=rename,
         leftrev=leftrev,
         leftnode=hex(leftnode),
         rightrev=rightrev,
         rightnode=hex(rightnode),
         comparison=comparison,
         **pycompat.strkwargs(webutil.commonentry(web.repo, ctx))))
 
     return web.res
 
 @webcommand('annotate')
 def annotate(web, req, tmpl):
     """
     /annotate/{revision}/{path}
     ---------------------------
 
     Show changeset information for each line in a file.
 
     The ``ignorews``, ``ignorewsamount``, ``ignorewseol``, and
     ``ignoreblanklines`` query string arguments have the same meaning as
     their ``[annotate]`` config equivalents. It uses the hgrc boolean
     parsing logic to interpret the value. e.g. ``0`` and ``false`` are
     false and ``1`` and ``true`` are true. If not defined, the server
     default settings are used.
 
     The ``fileannotate`` template is rendered.
     """
     fctx = webutil.filectx(web.repo, req)
     f = fctx.path()
     parity = paritygen(web.stripecount)
     ishead = fctx.filerev() in fctx.filelog().headrevs()
 
     # parents() is called once per line and several lines likely belong to
     # same revision. So it is worth caching.
     # TODO there are still redundant operations within basefilectx.parents()
     # and from the fctx.annotate() call itself that could be cached.
     parentscache = {}
     def parents(f):
         rev = f.rev()
         if rev not in parentscache:
             parentscache[rev] = []
             for p in f.parents():
                 entry = {
                     'node': p.hex(),
                     'rev': p.rev(),
                 }
                 parentscache[rev].append(entry)
 
         for p in parentscache[rev]:
             yield p
 
     def annotate(**map):
         if fctx.isbinary():
             mt = (mimetypes.guess_type(fctx.path())[0]
                   or 'application/octet-stream')
             lines = [((fctx.filectx(fctx.filerev()), 1), '(binary:%s)' % mt)]
         else:
             lines = webutil.annotate(req, fctx, web.repo.ui)
 
         previousrev = None
         blockparitygen = paritygen(1)
         for lineno, (aline, l) in enumerate(lines):
             f = aline.fctx
             rev = f.rev()
             if rev != previousrev:
                 blockhead = True
                 blockparity = next(blockparitygen)
             else:
                 blockhead = None
             previousrev = rev
             yield {"parity": next(parity),
                    "node": f.hex(),
                    "rev": rev,
                    "author": f.user(),
                    "parents": parents(f),
                    "desc": f.description(),
                    "extra": f.extra(),
                    "file": f.path(),
                    "blockhead": blockhead,
                    "blockparity": blockparity,
                    "targetline": aline.lineno,
                    "line": l,
                    "lineno": lineno + 1,
                    "lineid": "l%d" % (lineno + 1),
                    "linenumber": "% 6d" % (lineno + 1),
                    "revdate": f.date()}
 
     diffopts = webutil.difffeatureopts(req, web.repo.ui, 'annotate')
     diffopts = {k: getattr(diffopts, k) for k in diffopts.defaults}
 
     web.res.setbodygen(tmpl(
         'fileannotate',
         file=f,
         annotate=annotate,
         path=webutil.up(f),
         symrev=webutil.symrevorshortnode(req, fctx),
         rename=webutil.renamelink(fctx),
         permissions=fctx.manifest().flags(f),
         ishead=int(ishead),
         diffopts=diffopts,
         **pycompat.strkwargs(webutil.commonentry(web.repo, fctx))))
 
     return web.res
 
 @webcommand('filelog')
 def filelog(web, req, tmpl):
     """
     /filelog/{revision}/{path}
     --------------------------
 
     Show information about the history of a file in the repository.
 
     The ``revcount`` query string argument can be defined to control the
     maximum number of entries to show.
 
     The ``filelog`` template will be rendered.
     """
 
     try:
         fctx = webutil.filectx(web.repo, req)
         f = fctx.path()
         fl = fctx.filelog()
     except error.LookupError:
         f = webutil.cleanpath(web.repo, req.req.qsparams['file'])
         fl = web.repo.file(f)
         numrevs = len(fl)
         if not numrevs: # file doesn't exist at all
             raise
         rev = webutil.changectx(web.repo, req).rev()
         first = fl.linkrev(0)
         if rev < first: # current rev is from before file existed
             raise
         frev = numrevs - 1
         while fl.linkrev(frev) > rev:
             frev -= 1
         fctx = web.repo.filectx(f, fl.linkrev(frev))
 
     revcount = web.maxshortchanges
     if 'revcount' in req.req.qsparams:
         try:
             revcount = int(req.req.qsparams.get('revcount', revcount))
             revcount = max(revcount, 1)
             tmpl.defaults['sessionvars']['revcount'] = revcount
         except ValueError:
             pass
 
     lrange = webutil.linerange(req)
 
     lessvars = copy.copy(tmpl.defaults['sessionvars'])
     lessvars['revcount'] = max(revcount // 2, 1)
     morevars = copy.copy(tmpl.defaults['sessionvars'])
     morevars['revcount'] = revcount * 2
 
     patch = 'patch' in req.req.qsparams
     if patch:
         lessvars['patch'] = morevars['patch'] = req.req.qsparams['patch']
     descend = 'descend' in req.req.qsparams
     if descend:
         lessvars['descend'] = morevars['descend'] = req.req.qsparams['descend']
 
     count = fctx.filerev() + 1
     start = max(0, count - revcount) # first rev on this page
     end = min(count, start + revcount) # last rev on this page
     parity = paritygen(web.stripecount, offset=start - end)
 
     repo = web.repo
     revs = fctx.filelog().revs(start, end - 1)
     entries = []
 
     diffstyle = web.config('web', 'style')
     if 'style' in req.req.qsparams:
         diffstyle = req.req.qsparams['style']
 
     def diff(fctx, linerange=None):
         ctx = fctx.changectx()
         basectx = ctx.p1()
         path = fctx.path()
         return webutil.diffs(web, tmpl, ctx, basectx, [path], diffstyle,
                              linerange=linerange,
                              lineidprefix='%s-' % ctx.hex()[:12])
 
     linerange = None
     if lrange is not None:
         linerange = webutil.formatlinerange(*lrange)
         # deactivate numeric nav links when linerange is specified as this
         # would required a dedicated "revnav" class
         nav = None
         if descend:
             it = dagop.blockdescendants(fctx, *lrange)
         else:
             it = dagop.blockancestors(fctx, *lrange)
         for i, (c, lr) in enumerate(it, 1):
             diffs = None
             if patch:
                 diffs = diff(c, linerange=lr)
             # follow renames accross filtered (not in range) revisions
             path = c.path()
             entries.append(dict(
                 parity=next(parity),
                 filerev=c.rev(),
                 file=path,
                 diff=diffs,
                 linerange=webutil.formatlinerange(*lr),
                 **pycompat.strkwargs(webutil.commonentry(repo, c))))
             if i == revcount:
                 break
         lessvars['linerange'] = webutil.formatlinerange(*lrange)
         morevars['linerange'] = lessvars['linerange']
     else:
         for i in revs:
             iterfctx = fctx.filectx(i)
             diffs = None
             if patch:
                 diffs = diff(iterfctx)
             entries.append(dict(
                 parity=next(parity),
                 filerev=i,
                 file=f,
                 diff=diffs,
                 rename=webutil.renamelink(iterfctx),
                 **pycompat.strkwargs(webutil.commonentry(repo, iterfctx))))
         entries.reverse()
         revnav = webutil.filerevnav(web.repo, fctx.path())
         nav = revnav.gen(end - 1, revcount, count)
 
     latestentry = entries[:1]
 
     web.res.setbodygen(tmpl(
         'filelog',
         file=f,
         nav=nav,
         symrev=webutil.symrevorshortnode(req, fctx),
         entries=entries,
         descend=descend,
         patch=patch,
         latestentry=latestentry,
         linerange=linerange,
         revcount=revcount,
         morevars=morevars,
         lessvars=lessvars,
         **pycompat.strkwargs(webutil.commonentry(web.repo, fctx))))
 
     return web.res
 
 @webcommand('archive')
 def archive(web, req, tmpl):
     """
     /archive/{revision}.{format}[/{path}]
     -------------------------------------
 
     Obtain an archive of repository content.
 
     The content and type of the archive is defined by a URL path parameter.
     ``format`` is the file extension of the archive type to be generated. e.g.
     ``zip`` or ``tar.bz2``. Not all archive types may be allowed by your
     server configuration.
 
     The optional ``path`` URL parameter controls content to include in the
     archive. If omitted, every file in the specified revision is present in the
     archive. If included, only the specified file or contents of the specified
     directory will be included in the archive.
 
     No template is used for this handler. Raw, binary content is generated.
     """
 
     type_ = req.req.qsparams.get('type')
     allowed = web.configlist("web", "allow_archive")
     key = req.req.qsparams['node']
 
     if type_ not in web.archivespecs:
         msg = 'Unsupported archive type: %s' % type_
         raise ErrorResponse(HTTP_NOT_FOUND, msg)
 
     if not ((type_ in allowed or
              web.configbool("web", "allow" + type_))):
         msg = 'Archive type not allowed: %s' % type_
         raise ErrorResponse(HTTP_FORBIDDEN, msg)
 
     reponame = re.sub(br"\W+", "-", os.path.basename(web.reponame))
     cnode = web.repo.lookup(key)
     arch_version = key
     if cnode == key or key == 'tip':
         arch_version = short(cnode)
     name = "%s-%s" % (reponame, arch_version)
 
     ctx = webutil.changectx(web.repo, req)
     pats = []
     match = scmutil.match(ctx, [])
     file = req.req.qsparams.get('file')
     if file:
         pats = ['path:' + file]
         match = scmutil.match(ctx, pats, default='path')
         if pats:
             files = [f for f in ctx.manifest().keys() if match(f)]
             if not files:
                 raise ErrorResponse(HTTP_NOT_FOUND,
                     'file(s) not found: %s' % file)
 
     mimetype, artype, extension, encoding = web.archivespecs[type_]
     headers = [
         ('Content-Disposition', 'attachment; filename=%s%s' % (name, extension))
         ]
     if encoding:
         headers.append(('Content-Encoding', encoding))
     req.headers.extend(headers)
     req.respond(HTTP_OK, mimetype)
 
-    archival.archive(web.repo, req, cnode, artype, prefix=name,
+    bodyfh = requestmod.offsettrackingwriter(req.write)
+
+    archival.archive(web.repo, bodyfh, cnode, artype, prefix=name,
                      matchfn=match,
                      subrepos=web.configbool("web", "archivesubrepos"))
     return []
 
 
 @webcommand('static')
 def static(web, req, tmpl):
     fname = req.req.qsparams['file']
     # a repo owner may set web.static in .hg/hgrc to get any file
     # readable by the user running the CGI script
     static = web.config("web", "static", None, untrusted=False)
     if not static:
         tp = web.templatepath or templater.templatepaths()
         if isinstance(tp, str):
             tp = [tp]
         static = [os.path.join(p, 'static') for p in tp]
 
     staticfile(static, fname, web.res)
     return web.res
 
 @webcommand('graph')
 def graph(web, req, tmpl):
     """
     /graph[/{revision}]
     -------------------
 
     Show information about the graphical topology of the repository.
 
     Information rendered by this handler can be used to create visual
     representations of repository topology.
 
     The ``revision`` URL parameter controls the starting changeset. If it's
     absent, the default is ``tip``.
 
     The ``revcount`` query string argument can define the number of changesets
     to show information for.
 
     The ``graphtop`` query string argument can specify the starting changeset
     for producing ``jsdata`` variable that is used for rendering graph in
     JavaScript. By default it has the same value as ``revision``.
 
     This handler will render the ``graph`` template.
     """
 
     if 'node' in req.req.qsparams:
         ctx = webutil.changectx(web.repo, req)
         symrev = webutil.symrevorshortnode(req, ctx)
     else:
         ctx = web.repo['tip']
         symrev = 'tip'
     rev = ctx.rev()
 
     bg_height = 39
     revcount = web.maxshortchanges
     if 'revcount' in req.req.qsparams:
         try:
             revcount = int(req.req.qsparams.get('revcount', revcount))
             revcount = max(revcount, 1)
             tmpl.defaults['sessionvars']['revcount'] = revcount
         except ValueError:
             pass
 
     lessvars = copy.copy(tmpl.defaults['sessionvars'])
     lessvars['revcount'] = max(revcount // 2, 1)
     morevars = copy.copy(tmpl.defaults['sessionvars'])
     morevars['revcount'] = revcount * 2
 
     graphtop = req.req.qsparams.get('graphtop', ctx.hex())
     graphvars = copy.copy(tmpl.defaults['sessionvars'])
     graphvars['graphtop'] = graphtop
 
     count = len(web.repo)
     pos = rev
 
     uprev = min(max(0, count - 1), rev + revcount)
     downrev = max(0, rev - revcount)
     changenav = webutil.revnav(web.repo).gen(pos, revcount, count)
 
     tree = []
     nextentry = []
     lastrev = 0
     if pos != -1:
         allrevs = web.repo.changelog.revs(pos, 0)
         revs = []
         for i in allrevs:
             revs.append(i)
             if len(revs) >= revcount + 1:
                 break
 
         if len(revs) > revcount:
             nextentry = [webutil.commonentry(web.repo, web.repo[revs[-1]])]
             revs = revs[:-1]
 
         lastrev = revs[-1]
 
         # We have to feed a baseset to dagwalker as it is expecting smartset
         # object. This does not have a big impact on hgweb performance itself
         # since hgweb graphing code is not itself lazy yet.
         dag = graphmod.dagwalker(web.repo, smartset.baseset(revs))
         # As we said one line above... not lazy.
         tree = list(item for item in graphmod.colored(dag, web.repo)
                     if item[1] == graphmod.CHANGESET)
 
     def nodecurrent(ctx):
         wpnodes = web.repo.dirstate.parents()
         if wpnodes[1] == nullid:
             wpnodes = wpnodes[:1]
         if ctx.node() in wpnodes:
             return '@'
         return ''
 
     def nodesymbol(ctx):
         if ctx.obsolete():
             return 'x'
         elif ctx.isunstable():
             return '*'
         elif ctx.closesbranch():
             return '_'
         else:
             return 'o'
 
     def fulltree():
         pos = web.repo[graphtop].rev()
         tree = []
         if pos != -1:
             revs = web.repo.changelog.revs(pos, lastrev)
             dag = graphmod.dagwalker(web.repo, smartset.baseset(revs))
             tree = list(item for item in graphmod.colored(dag, web.repo)
                         if item[1] == graphmod.CHANGESET)
         return tree
 
     def jsdata():
         return [{'node': pycompat.bytestr(ctx),
                  'graphnode': nodecurrent(ctx) + nodesymbol(ctx),
                  'vertex': vtx,
                  'edges': edges}
                 for (id, type, ctx, vtx, edges) in fulltree()]
 
     def nodes():
         parity = paritygen(web.stripecount)
         for row, (id, type, ctx, vtx, edges) in enumerate(tree):
             entry = webutil.commonentry(web.repo, ctx)
             edgedata = [{'col': edge[0],
                          'nextcol': edge[1],
                          'color': (edge[2] - 1) % 6 + 1,
                          'width': edge[3],
                          'bcolor': edge[4]}
                         for edge in edges]
 
             entry.update({'col': vtx[0],
                           'color': (vtx[1] - 1) % 6 + 1,
                           'parity': next(parity),
                           'edges': edgedata,
                           'row': row,
                           'nextrow': row + 1})
 
             yield entry
 
     rows = len(tree)
 
     web.res.setbodygen(tmpl(
         'graph',
         rev=rev,
         symrev=symrev,
         revcount=revcount,
         uprev=uprev,
         lessvars=lessvars,
         morevars=morevars,
         downrev=downrev,
         graphvars=graphvars,
         rows=rows,
         bg_height=bg_height,
         changesets=count,
         nextentry=nextentry,
         jsdata=lambda **x: jsdata(),
         nodes=lambda **x: nodes(),
         node=ctx.hex(),
         changenav=changenav))
 
     return web.res
 
 def _getdoc(e):
     doc = e[0].__doc__
     if doc:
         doc = _(doc).partition('\n')[0]
     else:
         doc = _('(no help text available)')
     return doc
 
 @webcommand('help')
 def help(web, req, tmpl):
     """
     /help[/{topic}]
     ---------------
 
     Render help documentation.
 
     This web command is roughly equivalent to :hg:`help`. If a ``topic``
     is defined, that help topic will be rendered. If not, an index of
     available help topics will be rendered.
 
     The ``help`` template will be rendered when requesting help for a topic.
     ``helptopics`` will be rendered for the index of help topics.
     """
     from .. import commands, help as helpmod  # avoid cycle
 
     topicname = req.req.qsparams.get('node')
     if not topicname:
         def topics(**map):
             for entries, summary, _doc in helpmod.helptable:
                 yield {'topic': entries[0], 'summary': summary}
 
         early, other = [], []
         primary = lambda s: s.partition('|')[0]
         for c, e in commands.table.iteritems():
             doc = _getdoc(e)
             if 'DEPRECATED' in doc or c.startswith('debug'):
                 continue
             cmd = primary(c)
             if cmd.startswith('^'):
                 early.append((cmd[1:], doc))
             else:
                 other.append((cmd, doc))
 
         early.sort()
         other.sort()
 
         def earlycommands(**map):
             for c, doc in early:
                 yield {'topic': c, 'summary': doc}
 
         def othercommands(**map):
             for c, doc in other:
                 yield {'topic': c, 'summary': doc}
 
         web.res.setbodygen(tmpl(
             'helptopics',
             topics=topics,
             earlycommands=earlycommands,
             othercommands=othercommands,
             title='Index'))
         return web.res
 
     # Render an index of sub-topics.
     if topicname in helpmod.subtopics:
         topics = []
         for entries, summary, _doc in helpmod.subtopics[topicname]:
             topics.append({
                 'topic': '%s.%s' % (topicname, entries[0]),
                 'basename': entries[0],
                 'summary': summary,
             })
 
         web.res.setbodygen(tmpl(
             'helptopics',
             topics=topics,
             title=topicname,
             subindex=True))
         return web.res
 
     u = webutil.wsgiui.load()
     u.verbose = True
 
     # Render a page from a sub-topic.
     if '.' in topicname:
         # TODO implement support for rendering sections, like
         # `hg help` works.
         topic, subtopic = topicname.split('.', 1)
         if topic not in helpmod.subtopics:
             raise ErrorResponse(HTTP_NOT_FOUND)
     else:
         topic = topicname
         subtopic = None
 
     try:
         doc = helpmod.help_(u, commands, topic, subtopic=subtopic)
     except error.Abort:
         raise ErrorResponse(HTTP_NOT_FOUND)
 
     web.res.setbodygen(tmpl(
         'help',
         topic=topicname,
         doc=doc))
 
     return web.res
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = commands.values()