py3: use int instead of pycompat.long

pycompat.long is aliased to int. So this should have no change in
functionality.

Differential Revision: https://phab.mercurial-scm.org/D12338

hgext/convert/monotone.py

@@ -1,410 +1,410 b''
 # monotone.py - monotone support for the convert extension
 #
 #  Copyright 2008, 2009 Mikkel Fahnoe Jorgensen <[e-mail]> and
 #  others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import os
 import re
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial import (
     error,
     pycompat,
 )
 from mercurial.utils import dateutil
 
 from . import common
 
 
 class monotone_source(common.converter_source, common.commandline):
     def __init__(self, ui, repotype, path=None, revs=None):
         common.converter_source.__init__(self, ui, repotype, path, revs)
         if revs and len(revs) > 1:
             raise error.Abort(
                 _(
                     b'monotone source does not support specifying '
                     b'multiple revs'
                 )
             )
         common.commandline.__init__(self, ui, b'mtn')
 
         self.ui = ui
         self.path = path
         self.automatestdio = False
         self.revs = revs
 
         norepo = common.NoRepo(
             _(b"%s does not look like a monotone repository") % path
         )
         if not os.path.exists(os.path.join(path, b'_MTN')):
             # Could be a monotone repository (SQLite db file)
             try:
                 f = open(path, b'rb')
                 header = f.read(16)
                 f.close()
             except IOError:
                 header = b''
             if header != b'SQLite format 3\x00':
                 raise norepo
 
         # regular expressions for parsing monotone output
         space = br'\s*'
         name = br'\s+"((?:\\"|[^"])*)"\s*'
         value = name
         revision = br'\s+\[(\w+)\]\s*'
         lines = br'(?:.|\n)+'
 
         self.dir_re = re.compile(space + b"dir" + name)
         self.file_re = re.compile(
             space + b"file" + name + b"content" + revision
         )
         self.add_file_re = re.compile(
             space + b"add_file" + name + b"content" + revision
         )
         self.patch_re = re.compile(
             space + b"patch" + name + b"from" + revision + b"to" + revision
         )
         self.rename_re = re.compile(space + b"rename" + name + b"to" + name)
         self.delete_re = re.compile(space + b"delete" + name)
         self.tag_re = re.compile(space + b"tag" + name + b"revision" + revision)
         self.cert_re = re.compile(
             lines + space + b"name" + name + b"value" + value
         )
 
         attr = space + b"file" + lines + space + b"attr" + space
         self.attr_execute_re = re.compile(
             attr + b'"mtn:execute"' + space + b'"true"'
         )
 
         # cached data
         self.manifest_rev = None
         self.manifest = None
         self.files = None
         self.dirs = None
 
         common.checktool(b'mtn', abort=False)
 
     def mtnrun(self, *args, **kwargs):
         if self.automatestdio:
             return self.mtnrunstdio(*args, **kwargs)
         else:
             return self.mtnrunsingle(*args, **kwargs)
 
     def mtnrunsingle(self, *args, **kwargs):
         kwargs['d'] = self.path
         return self.run0(b'automate', *args, **kwargs)
 
     def mtnrunstdio(self, *args, **kwargs):
         # Prepare the command in automate stdio format
         kwargs = pycompat.byteskwargs(kwargs)
         command = []
         for k, v in kwargs.items():
             command.append(b"%d:%s" % (len(k), k))
             if v:
                 command.append(b"%d:%s" % (len(v), v))
         if command:
             command.insert(0, b'o')
             command.append(b'e')
 
         command.append(b'l')
         for arg in args:
             command.append(b"%d:%s" % (len(arg), arg))
         command.append(b'e')
         command = b''.join(command)
 
         self.ui.debug(b"mtn: sending '%s'\n" % command)
         self.mtnwritefp.write(command)
         self.mtnwritefp.flush()
 
         return self.mtnstdioreadcommandoutput(command)
 
     def mtnstdioreadpacket(self):
         read = None
         commandnbr = b''
         while read != b':':
             read = self.mtnreadfp.read(1)
             if not read:
                 raise error.Abort(_(b'bad mtn packet - no end of commandnbr'))
             commandnbr += read
         commandnbr = commandnbr[:-1]
 
         stream = self.mtnreadfp.read(1)
         if stream not in b'mewptl':
             raise error.Abort(
                 _(b'bad mtn packet - bad stream type %s') % stream
             )
 
         read = self.mtnreadfp.read(1)
         if read != b':':
             raise error.Abort(_(b'bad mtn packet - no divider before size'))
 
         read = None
         lengthstr = b''
         while read != b':':
             read = self.mtnreadfp.read(1)
             if not read:
                 raise error.Abort(_(b'bad mtn packet - no end of packet size'))
             lengthstr += read
         try:
-            length = pycompat.long(lengthstr[:-1])
+            length = int(lengthstr[:-1])
         except TypeError:
             raise error.Abort(
                 _(b'bad mtn packet - bad packet size %s') % lengthstr
             )
 
         read = self.mtnreadfp.read(length)
         if len(read) != length:
             raise error.Abort(
                 _(
                     b"bad mtn packet - unable to read full packet "
                     b"read %s of %s"
                 )
                 % (len(read), length)
             )
 
         return (commandnbr, stream, length, read)
 
     def mtnstdioreadcommandoutput(self, command):
         retval = []
         while True:
             commandnbr, stream, length, output = self.mtnstdioreadpacket()
             self.ui.debug(
                 b'mtn: read packet %s:%s:%d\n' % (commandnbr, stream, length)
             )
 
             if stream == b'l':
                 # End of command
                 if output != b'0':
                     raise error.Abort(
                         _(b"mtn command '%s' returned %s") % (command, output)
                     )
                 break
             elif stream in b'ew':
                 # Error, warning output
                 self.ui.warn(_(b'%s error:\n') % self.command)
                 self.ui.warn(output)
             elif stream == b'p':
                 # Progress messages
                 self.ui.debug(b'mtn: ' + output)
             elif stream == b'm':
                 # Main stream - command output
                 retval.append(output)
 
         return b''.join(retval)
 
     def mtnloadmanifest(self, rev):
         if self.manifest_rev == rev:
             return
         self.manifest = self.mtnrun(b"get_manifest_of", rev).split(b"\n\n")
         self.manifest_rev = rev
         self.files = {}
         self.dirs = {}
 
         for e in self.manifest:
             m = self.file_re.match(e)
             if m:
                 attr = b""
                 name = m.group(1)
                 node = m.group(2)
                 if self.attr_execute_re.match(e):
                     attr += b"x"
                 self.files[name] = (node, attr)
             m = self.dir_re.match(e)
             if m:
                 self.dirs[m.group(1)] = True
 
     def mtnisfile(self, name, rev):
         # a non-file could be a directory or a deleted or renamed file
         self.mtnloadmanifest(rev)
         return name in self.files
 
     def mtnisdir(self, name, rev):
         self.mtnloadmanifest(rev)
         return name in self.dirs
 
     def mtngetcerts(self, rev):
         certs = {
             b"author": b"<missing>",
             b"date": b"<missing>",
             b"changelog": b"<missing>",
             b"branch": b"<missing>",
         }
         certlist = self.mtnrun(b"certs", rev)
         # mtn < 0.45:
         #   key "[e-mail]"
         # mtn >= 0.45:
         #   key [ff58a7ffb771907c4ff68995eada1c4da068d328]
         certlist = re.split(br'\n\n {6}key ["\[]', certlist)
         for e in certlist:
             m = self.cert_re.match(e)
             if m:
                 name, value = m.groups()
                 value = value.replace(br'\"', b'"')
                 value = value.replace(br'\\', b'\\')
                 certs[name] = value
         # Monotone may have subsecond dates: 2005-02-05T09:39:12.364306
         # and all times are stored in UTC
         certs[b"date"] = certs[b"date"].split(b'.')[0] + b" UTC"
         return certs
 
     # implement the converter_source interface:
 
     def getheads(self):
         if not self.revs:
             return self.mtnrun(b"leaves").splitlines()
         else:
             return self.revs
 
     def getchanges(self, rev, full):
         if full:
             raise error.Abort(
                 _(b"convert from monotone does not support --full")
             )
         revision = self.mtnrun(b"get_revision", rev).split(b"\n\n")
         files = {}
         ignoremove = {}
         renameddirs = []
         copies = {}
         for e in revision:
             m = self.add_file_re.match(e)
             if m:
                 files[m.group(1)] = rev
                 ignoremove[m.group(1)] = rev
             m = self.patch_re.match(e)
             if m:
                 files[m.group(1)] = rev
             # Delete/rename is handled later when the convert engine
             # discovers an IOError exception from getfile,
             # but only if we add the "from" file to the list of changes.
             m = self.delete_re.match(e)
             if m:
                 files[m.group(1)] = rev
             m = self.rename_re.match(e)
             if m:
                 toname = m.group(2)
                 fromname = m.group(1)
                 if self.mtnisfile(toname, rev):
                     ignoremove[toname] = 1
                     copies[toname] = fromname
                     files[toname] = rev
                     files[fromname] = rev
                 elif self.mtnisdir(toname, rev):
                     renameddirs.append((fromname, toname))
 
         # Directory renames can be handled only once we have recorded
         # all new files
         for fromdir, todir in renameddirs:
             renamed = {}
             for tofile in self.files:
                 if tofile in ignoremove:
                     continue
                 if tofile.startswith(todir + b'/'):
                     renamed[tofile] = fromdir + tofile[len(todir) :]
                     # Avoid chained moves like:
                     # d1(/a) => d3/d1(/a)
                     # d2 => d3
                     ignoremove[tofile] = 1
             for tofile, fromfile in renamed.items():
                 self.ui.debug(
                     b"copying file in renamed directory from '%s' to '%s'"
                     % (fromfile, tofile),
                     b'\n',
                 )
                 files[tofile] = rev
                 copies[tofile] = fromfile
             for fromfile in renamed.values():
                 files[fromfile] = rev
 
         return (files.items(), copies, set())
 
     def getfile(self, name, rev):
         if not self.mtnisfile(name, rev):
             return None, None
         try:
             data = self.mtnrun(b"get_file_of", name, r=rev)
         except Exception:
             return None, None
         self.mtnloadmanifest(rev)
         node, attr = self.files.get(name, (None, b""))
         return data, attr
 
     def getcommit(self, rev):
         extra = {}
         certs = self.mtngetcerts(rev)
         if certs.get(b'suspend') == certs[b"branch"]:
             extra[b'close'] = b'1'
         dateformat = b"%Y-%m-%dT%H:%M:%S"
         return common.commit(
             author=certs[b"author"],
             date=dateutil.datestr(dateutil.strdate(certs[b"date"], dateformat)),
             desc=certs[b"changelog"],
             rev=rev,
             parents=self.mtnrun(b"parents", rev).splitlines(),
             branch=certs[b"branch"],
             extra=extra,
         )
 
     def gettags(self):
         tags = {}
         for e in self.mtnrun(b"tags").split(b"\n\n"):
             m = self.tag_re.match(e)
             if m:
                 tags[m.group(1)] = m.group(2)
         return tags
 
     def getchangedfiles(self, rev, i):
         # This function is only needed to support --filemap
         # ... and we don't support that
         raise NotImplementedError
 
     def before(self):
         # Check if we have a new enough version to use automate stdio
         try:
             versionstr = self.mtnrunsingle(b"interface_version")
             version = float(versionstr)
         except Exception:
             raise error.Abort(
                 _(b"unable to determine mtn automate interface version")
             )
 
         if version >= 12.0:
             self.automatestdio = True
             self.ui.debug(
                 b"mtn automate version %f - using automate stdio\n" % version
             )
 
             # launch the long-running automate stdio process
             self.mtnwritefp, self.mtnreadfp = self._run2(
                 b'automate', b'stdio', b'-d', self.path
             )
             # read the headers
             read = self.mtnreadfp.readline()
             if read != b'format-version: 2\n':
                 raise error.Abort(
                     _(b'mtn automate stdio header unexpected: %s') % read
                 )
             while read != b'\n':
                 read = self.mtnreadfp.readline()
                 if not read:
                     raise error.Abort(
                         _(
                             b"failed to reach end of mtn automate "
                             b"stdio headers"
                         )
                     )
         else:
             self.ui.debug(
                 b"mtn automate version %s - not using automate stdio "
                 b"(automate >= 12.0 - mtn >= 0.46 is needed)\n" % version
             )
 
     def after(self):
         if self.automatestdio:
             self.mtnwritefp.close()
             self.mtnwritefp = None
             self.mtnreadfp.close()
             self.mtnreadfp = None

hgext/remotefilelog/shallowutil.py

@@ -1,543 +1,543 b''
 # shallowutil.py -- remotefilelog utilities
 #
 # Copyright 2014 Facebook, Inc.
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 import collections
 import errno
 import os
 import stat
 import struct
 import tempfile
 
 from mercurial.i18n import _
 from mercurial.pycompat import open
 from mercurial.node import hex
 from mercurial import (
     error,
     pycompat,
     revlog,
     util,
 )
 from mercurial.utils import (
     hashutil,
     storageutil,
     stringutil,
 )
 from . import constants
 
 if not pycompat.iswindows:
     import grp
 
 
 def isenabled(repo):
     """returns whether the repository is remotefilelog enabled or not"""
     return constants.SHALLOWREPO_REQUIREMENT in repo.requirements
 
 
 def getcachekey(reponame, file, id):
     pathhash = hex(hashutil.sha1(file).digest())
     return os.path.join(reponame, pathhash[:2], pathhash[2:], id)
 
 
 def getlocalkey(file, id):
     pathhash = hex(hashutil.sha1(file).digest())
     return os.path.join(pathhash, id)
 
 
 def getcachepath(ui, allowempty=False):
     cachepath = ui.config(b"remotefilelog", b"cachepath")
     if not cachepath:
         if allowempty:
             return None
         else:
             raise error.Abort(
                 _(b"could not find config option remotefilelog.cachepath")
             )
     return util.expandpath(cachepath)
 
 
 def getcachepackpath(repo, category):
     cachepath = getcachepath(repo.ui)
     if category != constants.FILEPACK_CATEGORY:
         return os.path.join(cachepath, repo.name, b'packs', category)
     else:
         return os.path.join(cachepath, repo.name, b'packs')
 
 
 def getlocalpackpath(base, category):
     return os.path.join(base, b'packs', category)
 
 
 def createrevlogtext(text, copyfrom=None, copyrev=None):
     """returns a string that matches the revlog contents in a
     traditional revlog
     """
     meta = {}
     if copyfrom or text.startswith(b'\1\n'):
         if copyfrom:
             meta[b'copy'] = copyfrom
             meta[b'copyrev'] = copyrev
         text = storageutil.packmeta(meta, text)
 
     return text
 
 
 def parsemeta(text):
     """parse mercurial filelog metadata"""
     meta, size = storageutil.parsemeta(text)
     if text.startswith(b'\1\n'):
         s = text.index(b'\1\n', 2)
         text = text[s + 2 :]
     return meta or {}, text
 
 
 def sumdicts(*dicts):
     """Adds all the values of *dicts together into one dictionary. This assumes
     the values in *dicts are all summable.
 
     e.g. [{'a': 4', 'b': 2}, {'b': 3, 'c': 1}] -> {'a': 4, 'b': 5, 'c': 1}
     """
     result = collections.defaultdict(lambda: 0)
     for dict in dicts:
         for k, v in dict.items():
             result[k] += v
     return result
 
 
 def prefixkeys(dict, prefix):
     """Returns ``dict`` with ``prefix`` prepended to all its keys."""
     result = {}
     for k, v in dict.items():
         result[prefix + k] = v
     return result
 
 
 def reportpackmetrics(ui, prefix, *stores):
     dicts = [s.getmetrics() for s in stores]
     dict = prefixkeys(sumdicts(*dicts), prefix + b'_')
     ui.log(prefix + b"_packsizes", b"\n", **pycompat.strkwargs(dict))
 
 
 def _parsepackmeta(metabuf):
     """parse datapack meta, bytes (<metadata-list>) -> dict
 
     The dict contains raw content - both keys and values are strings.
     Upper-level business may want to convert some of them to other types like
     integers, on their own.
 
     raise ValueError if the data is corrupted
     """
     metadict = {}
     offset = 0
     buflen = len(metabuf)
     while buflen - offset >= 3:
         key = metabuf[offset : offset + 1]
         offset += 1
         metalen = struct.unpack_from(b'!H', metabuf, offset)[0]
         offset += 2
         if offset + metalen > buflen:
             raise ValueError(b'corrupted metadata: incomplete buffer')
         value = metabuf[offset : offset + metalen]
         metadict[key] = value
         offset += metalen
     if offset != buflen:
         raise ValueError(b'corrupted metadata: redundant data')
     return metadict
 
 
 def _buildpackmeta(metadict):
     """reverse of _parsepackmeta, dict -> bytes (<metadata-list>)
 
     The dict contains raw content - both keys and values are strings.
     Upper-level business may want to serialize some of other types (like
     integers) to strings before calling this function.
 
     raise ProgrammingError when metadata key is illegal, or ValueError if
     length limit is exceeded
     """
     metabuf = b''
     for k, v in sorted((metadict or {}).items()):
         if len(k) != 1:
             raise error.ProgrammingError(b'packmeta: illegal key: %s' % k)
         if len(v) > 0xFFFE:
             raise ValueError(
                 b'metadata value is too long: 0x%x > 0xfffe' % len(v)
             )
         metabuf += k
         metabuf += struct.pack(b'!H', len(v))
         metabuf += v
     # len(metabuf) is guaranteed representable in 4 bytes, because there are
     # only 256 keys, and for each value, len(value) <= 0xfffe.
     return metabuf
 
 
 _metaitemtypes = {
-    constants.METAKEYFLAG: (int, pycompat.long),
-    constants.METAKEYSIZE: (int, pycompat.long),
+    constants.METAKEYFLAG: (int, int),
+    constants.METAKEYSIZE: (int, int),
 }
 
 
 def buildpackmeta(metadict):
     """like _buildpackmeta, but typechecks metadict and normalize it.
 
     This means, METAKEYSIZE and METAKEYSIZE should have integers as values,
     and METAKEYFLAG will be dropped if its value is 0.
     """
     newmeta = {}
     for k, v in (metadict or {}).items():
         expectedtype = _metaitemtypes.get(k, (bytes,))
         if not isinstance(v, expectedtype):
             raise error.ProgrammingError(b'packmeta: wrong type of key %s' % k)
         # normalize int to binary buffer
         if int in expectedtype:
             # optimization: remove flag if it's 0 to save space
             if k == constants.METAKEYFLAG and v == 0:
                 continue
             v = int2bin(v)
         newmeta[k] = v
     return _buildpackmeta(newmeta)
 
 
 def parsepackmeta(metabuf):
     """like _parsepackmeta, but convert fields to desired types automatically.
 
     This means, METAKEYFLAG and METAKEYSIZE fields will be converted to
     integers.
     """
     metadict = _parsepackmeta(metabuf)
     for k, v in metadict.items():
         if k in _metaitemtypes and int in _metaitemtypes[k]:
             metadict[k] = bin2int(v)
     return metadict
 
 
 def int2bin(n):
     """convert a non-negative integer to raw binary buffer"""
     buf = bytearray()
     while n > 0:
         buf.insert(0, n & 0xFF)
         n >>= 8
     return bytes(buf)
 
 
 def bin2int(buf):
     """the reverse of int2bin, convert a binary buffer to an integer"""
     x = 0
     for b in bytearray(buf):
         x <<= 8
         x |= b
     return x
 
 
 class BadRemotefilelogHeader(error.StorageError):
     """Exception raised when parsing a remotefilelog blob header fails."""
 
 
 def parsesizeflags(raw):
     """given a remotefilelog blob, return (headersize, rawtextsize, flags)
 
     see remotefilelogserver.createfileblob for the format.
     raise RuntimeError if the content is illformed.
     """
     flags = revlog.REVIDX_DEFAULT_FLAGS
     size = None
     try:
         index = raw.index(b'\0')
     except ValueError:
         raise BadRemotefilelogHeader(
             "unexpected remotefilelog header: illegal format"
         )
     header = raw[:index]
     if header.startswith(b'v'):
         # v1 and above, header starts with 'v'
         if header.startswith(b'v1\n'):
             for s in header.split(b'\n'):
                 if s.startswith(constants.METAKEYSIZE):
                     size = int(s[len(constants.METAKEYSIZE) :])
                 elif s.startswith(constants.METAKEYFLAG):
                     flags = int(s[len(constants.METAKEYFLAG) :])
         else:
             raise BadRemotefilelogHeader(
                 b'unsupported remotefilelog header: %s' % header
             )
     else:
         # v0, str(int(size)) is the header
         size = int(header)
     if size is None:
         raise BadRemotefilelogHeader(
             "unexpected remotefilelog header: no size found"
         )
     return index + 1, size, flags
 
 
 def buildfileblobheader(size, flags, version=None):
     """return the header of a remotefilelog blob.
 
     see remotefilelogserver.createfileblob for the format.
     approximately the reverse of parsesizeflags.
 
     version could be 0 or 1, or None (auto decide).
     """
     # choose v0 if flags is empty, otherwise v1
     if version is None:
         version = int(bool(flags))
     if version == 1:
         header = b'v1\n%s%d\n%s%d' % (
             constants.METAKEYSIZE,
             size,
             constants.METAKEYFLAG,
             flags,
         )
     elif version == 0:
         if flags:
             raise error.ProgrammingError(b'fileblob v0 does not support flag')
         header = b'%d' % size
     else:
         raise error.ProgrammingError(b'unknown fileblob version %d' % version)
     return header
 
 
 def ancestormap(raw):
     offset, size, flags = parsesizeflags(raw)
     start = offset + size
 
     mapping = {}
     while start < len(raw):
         divider = raw.index(b'\0', start + 80)
 
         currentnode = raw[start : (start + 20)]
         p1 = raw[(start + 20) : (start + 40)]
         p2 = raw[(start + 40) : (start + 60)]
         linknode = raw[(start + 60) : (start + 80)]
         copyfrom = raw[(start + 80) : divider]
 
         mapping[currentnode] = (p1, p2, linknode, copyfrom)
         start = divider + 1
 
     return mapping
 
 
 def readfile(path):
     f = open(path, b'rb')
     try:
         result = f.read()
 
         # we should never have empty files
         if not result:
             os.remove(path)
             raise IOError(b"empty file: %s" % path)
 
         return result
     finally:
         f.close()
 
 
 def unlinkfile(filepath):
     if pycompat.iswindows:
         # On Windows, os.unlink cannnot delete readonly files
         os.chmod(filepath, stat.S_IWUSR)
     os.unlink(filepath)
 
 
 def renamefile(source, destination):
     if pycompat.iswindows:
         # On Windows, os.rename cannot rename readonly files
         # and cannot overwrite destination if it exists
         os.chmod(source, stat.S_IWUSR)
         if os.path.isfile(destination):
             os.chmod(destination, stat.S_IWUSR)
             os.unlink(destination)
 
     os.rename(source, destination)
 
 
 def writefile(path, content, readonly=False):
     dirname, filename = os.path.split(path)
     if not os.path.exists(dirname):
         try:
             os.makedirs(dirname)
         except OSError as ex:
             if ex.errno != errno.EEXIST:
                 raise
 
     fd, temp = tempfile.mkstemp(prefix=b'.%s-' % filename, dir=dirname)
     os.close(fd)
 
     try:
         f = util.posixfile(temp, b'wb')
         f.write(content)
         f.close()
 
         if readonly:
             mode = 0o444
         else:
             # tempfiles are created with 0o600, so we need to manually set the
             # mode.
             oldumask = os.umask(0)
             # there's no way to get the umask without modifying it, so set it
             # back
             os.umask(oldumask)
             mode = ~oldumask
 
         renamefile(temp, path)
         os.chmod(path, mode)
     except Exception:
         try:
             unlinkfile(temp)
         except OSError:
             pass
         raise
 
 
 def sortnodes(nodes, parentfunc):
     """Topologically sorts the nodes, using the parentfunc to find
     the parents of nodes."""
     nodes = set(nodes)
     childmap = {}
     parentmap = {}
     roots = []
 
     # Build a child and parent map
     for n in nodes:
         parents = [p for p in parentfunc(n) if p in nodes]
         parentmap[n] = set(parents)
         for p in parents:
             childmap.setdefault(p, set()).add(n)
         if not parents:
             roots.append(n)
 
     roots.sort()
     # Process roots, adding children to the queue as they become roots
     results = []
     while roots:
         n = roots.pop(0)
         results.append(n)
         if n in childmap:
             children = childmap[n]
             for c in children:
                 childparents = parentmap[c]
                 childparents.remove(n)
                 if len(childparents) == 0:
                     # insert at the beginning, that way child nodes
                     # are likely to be output immediately after their
                     # parents.  This gives better compression results.
                     roots.insert(0, c)
 
     return results
 
 
 def readexactly(stream, n):
     '''read n bytes from stream.read and abort if less was available'''
     s = stream.read(n)
     if len(s) < n:
         raise error.Abort(
             _(b"stream ended unexpectedly (got %d bytes, expected %d)")
             % (len(s), n)
         )
     return s
 
 
 def readunpack(stream, fmt):
     data = readexactly(stream, struct.calcsize(fmt))
     return struct.unpack(fmt, data)
 
 
 def readpath(stream):
     rawlen = readexactly(stream, constants.FILENAMESIZE)
     pathlen = struct.unpack(constants.FILENAMESTRUCT, rawlen)[0]
     return readexactly(stream, pathlen)
 
 
 def readnodelist(stream):
     rawlen = readexactly(stream, constants.NODECOUNTSIZE)
     nodecount = struct.unpack(constants.NODECOUNTSTRUCT, rawlen)[0]
     for i in pycompat.xrange(nodecount):
         yield readexactly(stream, constants.NODESIZE)
 
 
 def readpathlist(stream):
     rawlen = readexactly(stream, constants.PATHCOUNTSIZE)
     pathcount = struct.unpack(constants.PATHCOUNTSTRUCT, rawlen)[0]
     for i in pycompat.xrange(pathcount):
         yield readpath(stream)
 
 
 def getgid(groupname):
     try:
         gid = grp.getgrnam(pycompat.fsdecode(groupname)).gr_gid
         return gid
     except KeyError:
         return None
 
 
 def setstickygroupdir(path, gid, warn=None):
     if gid is None:
         return
     try:
         os.chown(path, -1, gid)
         os.chmod(path, 0o2775)
     except (IOError, OSError) as ex:
         if warn:
             warn(_(b'unable to chown/chmod on %s: %s\n') % (path, ex))
 
 
 def mkstickygroupdir(ui, path):
     """Creates the given directory (if it doesn't exist) and give it a
     particular group with setgid enabled."""
     gid = None
     groupname = ui.config(b"remotefilelog", b"cachegroup")
     if groupname:
         gid = getgid(groupname)
         if gid is None:
             ui.warn(_(b'unable to resolve group name: %s\n') % groupname)
 
     # we use a single stat syscall to test the existence and mode / group bit
     st = None
     try:
         st = os.stat(path)
     except OSError:
         pass
 
     if st:
         # exists
         if (st.st_mode & 0o2775) != 0o2775 or st.st_gid != gid:
             # permission needs to be fixed
             setstickygroupdir(path, gid, ui.warn)
         return
 
     oldumask = os.umask(0o002)
     try:
         missingdirs = [path]
         path = os.path.dirname(path)
         while path and not os.path.exists(path):
             missingdirs.append(path)
             path = os.path.dirname(path)
 
         for path in reversed(missingdirs):
             try:
                 os.mkdir(path)
             except OSError as ex:
                 if ex.errno != errno.EEXIST:
                     raise
 
         for path in missingdirs:
             setstickygroupdir(path, gid, ui.warn)
     finally:
         os.umask(oldumask)
 
 
 def getusername(ui):
     try:
         return stringutil.shortuser(ui.username())
     except Exception:
         return b'unknown'
 
 
 def getreponame(ui):
     reponame = ui.config(b'paths', b'default')
     if reponame:
         return os.path.basename(reponame)
     return b"unknown"

mercurial/formatter.py

@@ -1,870 +1,870 b''
 # formatter.py - generic output formatting for mercurial
 #
 # Copyright 2012 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 """Generic output formatting for Mercurial
 
 The formatter provides API to show data in various ways. The following
 functions should be used in place of ui.write():
 
 - fm.write() for unconditional output
 - fm.condwrite() to show some extra data conditionally in plain output
 - fm.context() to provide changectx to template output
 - fm.data() to provide extra data to JSON or template output
 - fm.plain() to show raw text that isn't provided to JSON or template output
 
 To show structured data (e.g. date tuples, dicts, lists), apply fm.format*()
 beforehand so the data is converted to the appropriate data type. Use
 fm.isplain() if you need to convert or format data conditionally which isn't
 supported by the formatter API.
 
 To build nested structure (i.e. a list of dicts), use fm.nested().
 
 See also https://www.mercurial-scm.org/wiki/GenericTemplatingPlan
 
 fm.condwrite() vs 'if cond:':
 
 In most cases, use fm.condwrite() so users can selectively show the data
 in template output. If it's costly to build data, use plain 'if cond:' with
 fm.write().
 
 fm.nested() vs fm.formatdict() (or fm.formatlist()):
 
 fm.nested() should be used to form a tree structure (a list of dicts of
 lists of dicts...) which can be accessed through template keywords, e.g.
 "{foo % "{bar % {...}} {baz % {...}}"}". On the other hand, fm.formatdict()
 exports a dict-type object to template, which can be accessed by e.g.
 "{get(foo, key)}" function.
 
 Doctest helper:
 
 >>> def show(fn, verbose=False, **opts):
 ...     import sys
 ...     from . import ui as uimod
 ...     ui = uimod.ui()
 ...     ui.verbose = verbose
 ...     ui.pushbuffer()
 ...     try:
 ...         return fn(ui, ui.formatter(pycompat.sysbytes(fn.__name__),
 ...                   pycompat.byteskwargs(opts)))
 ...     finally:
 ...         print(pycompat.sysstr(ui.popbuffer()), end='')
 
 Basic example:
 
 >>> def files(ui, fm):
 ...     files = [(b'foo', 123, (0, 0)), (b'bar', 456, (1, 0))]
 ...     for f in files:
 ...         fm.startitem()
 ...         fm.write(b'path', b'%s', f[0])
 ...         fm.condwrite(ui.verbose, b'date', b'  %s',
 ...                      fm.formatdate(f[2], b'%Y-%m-%d %H:%M:%S'))
 ...         fm.data(size=f[1])
 ...         fm.plain(b'\\n')
 ...     fm.end()
 >>> show(files)
 foo
 bar
 >>> show(files, verbose=True)
 foo  1970-01-01 00:00:00
 bar  1970-01-01 00:00:01
 >>> show(files, template=b'json')
 [
  {
   "date": [0, 0],
   "path": "foo",
   "size": 123
  },
  {
   "date": [1, 0],
   "path": "bar",
   "size": 456
  }
 ]
 >>> show(files, template=b'path: {path}\\ndate: {date|rfc3339date}\\n')
 path: foo
 date: 1970-01-01T00:00:00+00:00
 path: bar
 date: 1970-01-01T00:00:01+00:00
 
 Nested example:
 
 >>> def subrepos(ui, fm):
 ...     fm.startitem()
 ...     fm.write(b'reponame', b'[%s]\\n', b'baz')
 ...     files(ui, fm.nested(b'files', tmpl=b'{reponame}'))
 ...     fm.end()
 >>> show(subrepos)
 [baz]
 foo
 bar
 >>> show(subrepos, template=b'{reponame}: {join(files % "{path}", ", ")}\\n')
 baz: foo, bar
 """
 
 
 import contextlib
 import itertools
 import os
 import pickle
 
 from .i18n import _
 from .node import (
     hex,
     short,
 )
 from .thirdparty import attr
 
 from . import (
     error,
     pycompat,
     templatefilters,
     templatekw,
     templater,
     templateutil,
     util,
 )
 from .utils import (
     cborutil,
     dateutil,
     stringutil,
 )
 
 
 def isprintable(obj):
     """Check if the given object can be directly passed in to formatter's
     write() and data() functions
 
     Returns False if the object is unsupported or must be pre-processed by
     formatdate(), formatdict(), or formatlist().
     """
-    return isinstance(obj, (type(None), bool, int, pycompat.long, float, bytes))
+    return isinstance(obj, (type(None), bool, int, int, float, bytes))
 
 
 class _nullconverter(object):
     '''convert non-primitive data types to be processed by formatter'''
 
     # set to True if context object should be stored as item
     storecontext = False
 
     @staticmethod
     def wrapnested(data, tmpl, sep):
         '''wrap nested data by appropriate type'''
         return data
 
     @staticmethod
     def formatdate(date, fmt):
         '''convert date tuple to appropriate format'''
         # timestamp can be float, but the canonical form should be int
         ts, tz = date
         return (int(ts), tz)
 
     @staticmethod
     def formatdict(data, key, value, fmt, sep):
         '''convert dict or key-value pairs to appropriate dict format'''
         # use plain dict instead of util.sortdict so that data can be
         # serialized as a builtin dict in pickle output
         return dict(data)
 
     @staticmethod
     def formatlist(data, name, fmt, sep):
         '''convert iterable to appropriate list format'''
         return list(data)
 
 
 class baseformatter(object):
 
     # set to True if the formater output a strict format that does not support
     # arbitrary output in the stream.
     strict_format = False
 
     def __init__(self, ui, topic, opts, converter):
         self._ui = ui
         self._topic = topic
         self._opts = opts
         self._converter = converter
         self._item = None
         # function to convert node to string suitable for this output
         self.hexfunc = hex
 
     def __enter__(self):
         return self
 
     def __exit__(self, exctype, excvalue, traceback):
         if exctype is None:
             self.end()
 
     def _showitem(self):
         '''show a formatted item once all data is collected'''
 
     def startitem(self):
         '''begin an item in the format list'''
         if self._item is not None:
             self._showitem()
         self._item = {}
 
     def formatdate(self, date, fmt=b'%a %b %d %H:%M:%S %Y %1%2'):
         '''convert date tuple to appropriate format'''
         return self._converter.formatdate(date, fmt)
 
     def formatdict(self, data, key=b'key', value=b'value', fmt=None, sep=b' '):
         '''convert dict or key-value pairs to appropriate dict format'''
         return self._converter.formatdict(data, key, value, fmt, sep)
 
     def formatlist(self, data, name, fmt=None, sep=b' '):
         '''convert iterable to appropriate list format'''
         # name is mandatory argument for now, but it could be optional if
         # we have default template keyword, e.g. {item}
         return self._converter.formatlist(data, name, fmt, sep)
 
     def context(self, **ctxs):
         '''insert context objects to be used to render template keywords'''
         ctxs = pycompat.byteskwargs(ctxs)
         assert all(k in {b'repo', b'ctx', b'fctx'} for k in ctxs)
         if self._converter.storecontext:
             # populate missing resources in fctx -> ctx -> repo order
             if b'fctx' in ctxs and b'ctx' not in ctxs:
                 ctxs[b'ctx'] = ctxs[b'fctx'].changectx()
             if b'ctx' in ctxs and b'repo' not in ctxs:
                 ctxs[b'repo'] = ctxs[b'ctx'].repo()
             self._item.update(ctxs)
 
     def datahint(self):
         '''set of field names to be referenced'''
         return set()
 
     def data(self, **data):
         '''insert data into item that's not shown in default output'''
         data = pycompat.byteskwargs(data)
         self._item.update(data)
 
     def write(self, fields, deftext, *fielddata, **opts):
         '''do default text output while assigning data to item'''
         fieldkeys = fields.split()
         assert len(fieldkeys) == len(fielddata), (fieldkeys, fielddata)
         self._item.update(zip(fieldkeys, fielddata))
 
     def condwrite(self, cond, fields, deftext, *fielddata, **opts):
         '''do conditional write (primarily for plain formatter)'''
         fieldkeys = fields.split()
         assert len(fieldkeys) == len(fielddata)
         self._item.update(zip(fieldkeys, fielddata))
 
     def plain(self, text, **opts):
         '''show raw text for non-templated mode'''
 
     def isplain(self):
         '''check for plain formatter usage'''
         return False
 
     def nested(self, field, tmpl=None, sep=b''):
         '''sub formatter to store nested data in the specified field'''
         data = []
         self._item[field] = self._converter.wrapnested(data, tmpl, sep)
         return _nestedformatter(self._ui, self._converter, data)
 
     def end(self):
         '''end output for the formatter'''
         if self._item is not None:
             self._showitem()
 
 
 def nullformatter(ui, topic, opts):
     '''formatter that prints nothing'''
     return baseformatter(ui, topic, opts, converter=_nullconverter)
 
 
 class _nestedformatter(baseformatter):
     '''build sub items and store them in the parent formatter'''
 
     def __init__(self, ui, converter, data):
         baseformatter.__init__(
             self, ui, topic=b'', opts={}, converter=converter
         )
         self._data = data
 
     def _showitem(self):
         self._data.append(self._item)
 
 
 def _iteritems(data):
     '''iterate key-value pairs in stable order'''
     if isinstance(data, dict):
         return sorted(data.items())
     return data
 
 
 class _plainconverter(object):
     '''convert non-primitive data types to text'''
 
     storecontext = False
 
     @staticmethod
     def wrapnested(data, tmpl, sep):
         raise error.ProgrammingError(b'plainformatter should never be nested')
 
     @staticmethod
     def formatdate(date, fmt):
         '''stringify date tuple in the given format'''
         return dateutil.datestr(date, fmt)
 
     @staticmethod
     def formatdict(data, key, value, fmt, sep):
         '''stringify key-value pairs separated by sep'''
         prefmt = pycompat.identity
         if fmt is None:
             fmt = b'%s=%s'
             prefmt = pycompat.bytestr
         return sep.join(
             fmt % (prefmt(k), prefmt(v)) for k, v in _iteritems(data)
         )
 
     @staticmethod
     def formatlist(data, name, fmt, sep):
         '''stringify iterable separated by sep'''
         prefmt = pycompat.identity
         if fmt is None:
             fmt = b'%s'
             prefmt = pycompat.bytestr
         return sep.join(fmt % prefmt(e) for e in data)
 
 
 class plainformatter(baseformatter):
     '''the default text output scheme'''
 
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _plainconverter)
         if ui.debugflag:
             self.hexfunc = hex
         else:
             self.hexfunc = short
         if ui is out:
             self._write = ui.write
         else:
             self._write = lambda s, **opts: out.write(s)
 
     def startitem(self):
         pass
 
     def data(self, **data):
         pass
 
     def write(self, fields, deftext, *fielddata, **opts):
         self._write(deftext % fielddata, **opts)
 
     def condwrite(self, cond, fields, deftext, *fielddata, **opts):
         '''do conditional write'''
         if cond:
             self._write(deftext % fielddata, **opts)
 
     def plain(self, text, **opts):
         self._write(text, **opts)
 
     def isplain(self):
         return True
 
     def nested(self, field, tmpl=None, sep=b''):
         # nested data will be directly written to ui
         return self
 
     def end(self):
         pass
 
 
 class debugformatter(baseformatter):
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._out.write(b"%s = [\n" % self._topic)
 
     def _showitem(self):
         self._out.write(
             b'    %s,\n' % stringutil.pprint(self._item, indent=4, level=1)
         )
 
     def end(self):
         baseformatter.end(self)
         self._out.write(b"]\n")
 
 
 class pickleformatter(baseformatter):
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._data = []
 
     def _showitem(self):
         self._data.append(self._item)
 
     def end(self):
         baseformatter.end(self)
         self._out.write(pickle.dumps(self._data))
 
 
 class cborformatter(baseformatter):
     '''serialize items as an indefinite-length CBOR array'''
 
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._out.write(cborutil.BEGIN_INDEFINITE_ARRAY)
 
     def _showitem(self):
         self._out.write(b''.join(cborutil.streamencode(self._item)))
 
     def end(self):
         baseformatter.end(self)
         self._out.write(cborutil.BREAK)
 
 
 class jsonformatter(baseformatter):
 
     strict_format = True
 
     def __init__(self, ui, out, topic, opts):
         baseformatter.__init__(self, ui, topic, opts, _nullconverter)
         self._out = out
         self._out.write(b"[")
         self._first = True
 
     def _showitem(self):
         if self._first:
             self._first = False
         else:
             self._out.write(b",")
 
         self._out.write(b"\n {\n")
         first = True
         for k, v in sorted(self._item.items()):
             if first:
                 first = False
             else:
                 self._out.write(b",\n")
             u = templatefilters.json(v, paranoid=False)
             self._out.write(b'  "%s": %s' % (k, u))
         self._out.write(b"\n }")
 
     def end(self):
         baseformatter.end(self)
         self._out.write(b"\n]\n")
 
 
 class _templateconverter(object):
     '''convert non-primitive data types to be processed by templater'''
 
     storecontext = True
 
     @staticmethod
     def wrapnested(data, tmpl, sep):
         '''wrap nested data by templatable type'''
         return templateutil.mappinglist(data, tmpl=tmpl, sep=sep)
 
     @staticmethod
     def formatdate(date, fmt):
         '''return date tuple'''
         return templateutil.date(date)
 
     @staticmethod
     def formatdict(data, key, value, fmt, sep):
         '''build object that can be evaluated as either plain string or dict'''
         data = util.sortdict(_iteritems(data))
 
         def f():
             yield _plainconverter.formatdict(data, key, value, fmt, sep)
 
         return templateutil.hybriddict(
             data, key=key, value=value, fmt=fmt, gen=f
         )
 
     @staticmethod
     def formatlist(data, name, fmt, sep):
         '''build object that can be evaluated as either plain string or list'''
         data = list(data)
 
         def f():
             yield _plainconverter.formatlist(data, name, fmt, sep)
 
         return templateutil.hybridlist(data, name=name, fmt=fmt, gen=f)
 
 
 class templateformatter(baseformatter):
     def __init__(self, ui, out, topic, opts, spec, overridetemplates=None):
         baseformatter.__init__(self, ui, topic, opts, _templateconverter)
         self._out = out
         self._tref = spec.ref
         self._t = loadtemplater(
             ui,
             spec,
             defaults=templatekw.keywords,
             resources=templateresources(ui),
             cache=templatekw.defaulttempl,
         )
         if overridetemplates:
             self._t.cache.update(overridetemplates)
         self._parts = templatepartsmap(
             spec, self._t, [b'docheader', b'docfooter', b'separator']
         )
         self._counter = itertools.count()
         self._renderitem(b'docheader', {})
 
     def _showitem(self):
         item = self._item.copy()
         item[b'index'] = index = next(self._counter)
         if index > 0:
             self._renderitem(b'separator', {})
         self._renderitem(self._tref, item)
 
     def _renderitem(self, part, item):
         if part not in self._parts:
             return
         ref = self._parts[part]
         # None can't be put in the mapping dict since it means <unset>
         for k, v in item.items():
             if v is None:
                 item[k] = templateutil.wrappedvalue(v)
         self._out.write(self._t.render(ref, item))
 
     @util.propertycache
     def _symbolsused(self):
         return self._t.symbolsused(self._tref)
 
     def datahint(self):
         '''set of field names to be referenced from the template'''
         return self._symbolsused[0]
 
     def end(self):
         baseformatter.end(self)
         self._renderitem(b'docfooter', {})
 
 
 @attr.s(frozen=True)
 class templatespec(object):
     ref = attr.ib()
     tmpl = attr.ib()
     mapfile = attr.ib()
     refargs = attr.ib(default=None)
     fp = attr.ib(default=None)
 
 
 def empty_templatespec():
     return templatespec(None, None, None)
 
 
 def reference_templatespec(ref, refargs=None):
     return templatespec(ref, None, None, refargs)
 
 
 def literal_templatespec(tmpl):
     assert not isinstance(tmpl, str), b'tmpl must not be a str'
     return templatespec(b'', tmpl, None)
 
 
 def mapfile_templatespec(topic, mapfile, fp=None):
     return templatespec(topic, None, mapfile, fp=fp)
 
 
 def lookuptemplate(ui, topic, tmpl):
     """Find the template matching the given -T/--template spec 'tmpl'
 
     'tmpl' can be any of the following:
 
      - a literal template (e.g. '{rev}')
      - a reference to built-in template (i.e. formatter)
      - a map-file name or path (e.g. 'changelog')
      - a reference to [templates] in config file
      - a path to raw template file
 
     A map file defines a stand-alone template environment. If a map file
     selected, all templates defined in the file will be loaded, and the
     template matching the given topic will be rendered. Aliases won't be
     loaded from user config, but from the map file.
 
     If no map file selected, all templates in [templates] section will be
     available as well as aliases in [templatealias].
     """
 
     if not tmpl:
         return empty_templatespec()
 
     # looks like a literal template?
     if b'{' in tmpl:
         return literal_templatespec(tmpl)
 
     # a reference to built-in (formatter) template
     if tmpl in {b'cbor', b'json', b'pickle', b'debug'}:
         return reference_templatespec(tmpl)
 
     # a function-style reference to built-in template
     func, fsep, ftail = tmpl.partition(b'(')
     if func in {b'cbor', b'json'} and fsep and ftail.endswith(b')'):
         templater.parseexpr(tmpl)  # make sure syntax errors are confined
         return reference_templatespec(func, refargs=ftail[:-1])
 
     # perhaps a stock style?
     if not os.path.split(tmpl)[0]:
         (mapname, fp) = templater.try_open_template(
             b'map-cmdline.' + tmpl
         ) or templater.try_open_template(tmpl)
         if mapname:
             return mapfile_templatespec(topic, mapname, fp)
 
     # perhaps it's a reference to [templates]
     if ui.config(b'templates', tmpl):
         return reference_templatespec(tmpl)
 
     if tmpl == b'list':
         ui.write(_(b"available styles: %s\n") % templater.stylelist())
         raise error.Abort(_(b"specify a template"))
 
     # perhaps it's a path to a map or a template
     if (b'/' in tmpl or b'\\' in tmpl) and os.path.isfile(tmpl):
         # is it a mapfile for a style?
         if os.path.basename(tmpl).startswith(b"map-"):
             return mapfile_templatespec(topic, os.path.realpath(tmpl))
         with util.posixfile(tmpl, b'rb') as f:
             tmpl = f.read()
         return literal_templatespec(tmpl)
 
     # constant string?
     return literal_templatespec(tmpl)
 
 
 def templatepartsmap(spec, t, partnames):
     """Create a mapping of {part: ref}"""
     partsmap = {spec.ref: spec.ref}  # initial ref must exist in t
     if spec.mapfile:
         partsmap.update((p, p) for p in partnames if p in t)
     elif spec.ref:
         for part in partnames:
             ref = b'%s:%s' % (spec.ref, part)  # select config sub-section
             if ref in t:
                 partsmap[part] = ref
     return partsmap
 
 
 def loadtemplater(ui, spec, defaults=None, resources=None, cache=None):
     """Create a templater from either a literal template or loading from
     a map file"""
     assert not (spec.tmpl and spec.mapfile)
     if spec.mapfile:
         return templater.templater.frommapfile(
             spec.mapfile,
             spec.fp,
             defaults=defaults,
             resources=resources,
             cache=cache,
         )
     return maketemplater(
         ui, spec.tmpl, defaults=defaults, resources=resources, cache=cache
     )
 
 
 def maketemplater(ui, tmpl, defaults=None, resources=None, cache=None):
     """Create a templater from a string template 'tmpl'"""
     aliases = ui.configitems(b'templatealias')
     t = templater.templater(
         defaults=defaults, resources=resources, cache=cache, aliases=aliases
     )
     t.cache.update(
         (k, templater.unquotestring(v)) for k, v in ui.configitems(b'templates')
     )
     if tmpl:
         t.cache[b''] = tmpl
     return t
 
 
 # marker to denote a resource to be loaded on demand based on mapping values
 # (e.g. (ctx, path) -> fctx)
 _placeholder = object()
 
 
 class templateresources(templater.resourcemapper):
     """Resource mapper designed for the default templatekw and function"""
 
     def __init__(self, ui, repo=None):
         self._resmap = {
             b'cache': {},  # for templatekw/funcs to store reusable data
             b'repo': repo,
             b'ui': ui,
         }
 
     def availablekeys(self, mapping):
         return {
             k for k in self.knownkeys() if self._getsome(mapping, k) is not None
         }
 
     def knownkeys(self):
         return {b'cache', b'ctx', b'fctx', b'repo', b'revcache', b'ui'}
 
     def lookup(self, mapping, key):
         if key not in self.knownkeys():
             return None
         v = self._getsome(mapping, key)
         if v is _placeholder:
             v = mapping[key] = self._loadermap[key](self, mapping)
         return v
 
     def populatemap(self, context, origmapping, newmapping):
         mapping = {}
         if self._hasnodespec(newmapping):
             mapping[b'revcache'] = {}  # per-ctx cache
         if self._hasnodespec(origmapping) and self._hasnodespec(newmapping):
             orignode = templateutil.runsymbol(context, origmapping, b'node')
             mapping[b'originalnode'] = orignode
         # put marker to override 'ctx'/'fctx' in mapping if any, and flag
         # its existence to be reported by availablekeys()
         if b'ctx' not in newmapping and self._hasliteral(newmapping, b'node'):
             mapping[b'ctx'] = _placeholder
         if b'fctx' not in newmapping and self._hasliteral(newmapping, b'path'):
             mapping[b'fctx'] = _placeholder
         return mapping
 
     def _getsome(self, mapping, key):
         v = mapping.get(key)
         if v is not None:
             return v
         return self._resmap.get(key)
 
     def _hasliteral(self, mapping, key):
         """Test if a literal value is set or unset in the given mapping"""
         return key in mapping and not callable(mapping[key])
 
     def _getliteral(self, mapping, key):
         """Return value of the given name if it is a literal"""
         v = mapping.get(key)
         if callable(v):
             return None
         return v
 
     def _hasnodespec(self, mapping):
         """Test if context revision is set or unset in the given mapping"""
         return b'node' in mapping or b'ctx' in mapping
 
     def _loadctx(self, mapping):
         repo = self._getsome(mapping, b'repo')
         node = self._getliteral(mapping, b'node')
         if repo is None or node is None:
             return
         try:
             return repo[node]
         except error.RepoLookupError:
             return None  # maybe hidden/non-existent node
 
     def _loadfctx(self, mapping):
         ctx = self._getsome(mapping, b'ctx')
         path = self._getliteral(mapping, b'path')
         if ctx is None or path is None:
             return None
         try:
             return ctx[path]
         except error.LookupError:
             return None  # maybe removed file?
 
     _loadermap = {
         b'ctx': _loadctx,
         b'fctx': _loadfctx,
     }
 
 
 def _internaltemplateformatter(
     ui,
     out,
     topic,
     opts,
     spec,
     tmpl,
     docheader=b'',
     docfooter=b'',
     separator=b'',
 ):
     """Build template formatter that handles customizable built-in templates
     such as -Tjson(...)"""
     templates = {spec.ref: tmpl}
     if docheader:
         templates[b'%s:docheader' % spec.ref] = docheader
     if docfooter:
         templates[b'%s:docfooter' % spec.ref] = docfooter
     if separator:
         templates[b'%s:separator' % spec.ref] = separator
     return templateformatter(
         ui, out, topic, opts, spec, overridetemplates=templates
     )
 
 
 def formatter(ui, out, topic, opts):
     spec = lookuptemplate(ui, topic, opts.get(b'template', b''))
     if spec.ref == b"cbor" and spec.refargs is not None:
         return _internaltemplateformatter(
             ui,
             out,
             topic,
             opts,
             spec,
             tmpl=b'{dict(%s)|cbor}' % spec.refargs,
             docheader=cborutil.BEGIN_INDEFINITE_ARRAY,
             docfooter=cborutil.BREAK,
         )
     elif spec.ref == b"cbor":
         return cborformatter(ui, out, topic, opts)
     elif spec.ref == b"json" and spec.refargs is not None:
         return _internaltemplateformatter(
             ui,
             out,
             topic,
             opts,
             spec,
             tmpl=b'{dict(%s)|json}' % spec.refargs,
             docheader=b'[\n ',
             docfooter=b'\n]\n',
             separator=b',\n ',
         )
     elif spec.ref == b"json":
         return jsonformatter(ui, out, topic, opts)
     elif spec.ref == b"pickle":
         assert spec.refargs is None, r'function-style not supported'
         return pickleformatter(ui, out, topic, opts)
     elif spec.ref == b"debug":
         assert spec.refargs is None, r'function-style not supported'
         return debugformatter(ui, out, topic, opts)
     elif spec.ref or spec.tmpl or spec.mapfile:
         assert spec.refargs is None, r'function-style not supported'
         return templateformatter(ui, out, topic, opts, spec)
     # developer config: ui.formatdebug
     elif ui.configbool(b'ui', b'formatdebug'):
         return debugformatter(ui, out, topic, opts)
     # deprecated config: ui.formatjson
     elif ui.configbool(b'ui', b'formatjson'):
         return jsonformatter(ui, out, topic, opts)
     return plainformatter(ui, out, topic, opts)
 
 
 @contextlib.contextmanager
 def openformatter(ui, filename, topic, opts):
     """Create a formatter that writes outputs to the specified file
 
     Must be invoked using the 'with' statement.
     """
     with util.posixfile(filename, b'wb') as out:
         with formatter(ui, out, topic, opts) as fm:
             yield fm
 
 
 @contextlib.contextmanager
 def _neverending(fm):
     yield fm
 
 
 def maybereopen(fm, filename):
     """Create a formatter backed by file if filename specified, else return
     the given formatter
 
     Must be invoked using the 'with' statement. This will never call fm.end()
     of the given formatter.
     """
     if filename:
         return openformatter(fm._ui, filename, fm._topic, fm._opts)
     else:
         return _neverending(fm)

mercurial/templatefilters.py

@@ -1,556 +1,556 b''
 # templatefilters.py - common template expansion filters
 #
 # Copyright 2005-2008 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import os
 import re
 import time
 
 from .i18n import _
 from .node import hex
 from . import (
     encoding,
     error,
     pycompat,
     registrar,
     smartset,
     templateutil,
     url,
     util,
 )
 from .utils import (
     cborutil,
     dateutil,
     stringutil,
 )
 
 urlerr = util.urlerr
 urlreq = util.urlreq
 
 # filters are callables like:
 #   fn(obj)
 # with:
 #   obj - object to be filtered (text, date, list and so on)
 filters = {}
 
 templatefilter = registrar.templatefilter(filters)
 
 
 @templatefilter(b'addbreaks', intype=bytes)
 def addbreaks(text):
     """Any text. Add an XHTML "<br />" tag before the end of
     every line except the last.
     """
     return text.replace(b'\n', b'<br/>\n')
 
 
 agescales = [
     (b"year", 3600 * 24 * 365, b'Y'),
     (b"month", 3600 * 24 * 30, b'M'),
     (b"week", 3600 * 24 * 7, b'W'),
     (b"day", 3600 * 24, b'd'),
     (b"hour", 3600, b'h'),
     (b"minute", 60, b'm'),
     (b"second", 1, b's'),
 ]
 
 
 @templatefilter(b'age', intype=templateutil.date)
 def age(date, abbrev=False):
     """Date. Returns a human-readable date/time difference between the
     given date/time and the current date/time.
     """
 
     def plural(t, c):
         if c == 1:
             return t
         return t + b"s"
 
     def fmt(t, c, a):
         if abbrev:
             return b"%d%s" % (c, a)
         return b"%d %s" % (c, plural(t, c))
 
     now = time.time()
     then = date[0]
     future = False
     if then > now:
         future = True
         delta = max(1, int(then - now))
         if delta > agescales[0][1] * 30:
             return b'in the distant future'
     else:
         delta = max(1, int(now - then))
         if delta > agescales[0][1] * 2:
             return dateutil.shortdate(date)
 
     for t, s, a in agescales:
         n = delta // s
         if n >= 2 or s == 1:
             if future:
                 return b'%s from now' % fmt(t, n, a)
             return b'%s ago' % fmt(t, n, a)
 
 
 @templatefilter(b'basename', intype=bytes)
 def basename(path):
     """Any text. Treats the text as a path, and returns the last
     component of the path after splitting by the path separator.
     For example, "foo/bar/baz" becomes "baz" and "foo/bar//" becomes "".
     """
     return os.path.basename(path)
 
 
 def _tocborencodable(obj):
     if isinstance(obj, smartset.abstractsmartset):
         return list(obj)
     return obj
 
 
 @templatefilter(b'cbor')
 def cbor(obj):
     """Any object. Serializes the object to CBOR bytes."""
     # cborutil is stricter about type than json() filter
     obj = pycompat.rapply(_tocborencodable, obj)
     return b''.join(cborutil.streamencode(obj))
 
 
 @templatefilter(b'commondir')
 def commondir(filelist):
     """List of text. Treats each list item as file name with /
     as path separator and returns the longest common directory
     prefix shared by all list items.
     Returns the empty string if no common prefix exists.
 
     The list items are not normalized, i.e. "foo/../bar" is handled as
     file "bar" in the directory "foo/..". Leading slashes are ignored.
 
     For example, ["foo/bar/baz", "foo/baz/bar"] becomes "foo" and
     ["foo/bar", "baz"] becomes "".
     """
 
     def common(a, b):
         if len(a) > len(b):
             a = b[: len(a)]
         elif len(b) > len(a):
             b = b[: len(a)]
         if a == b:
             return a
         for i in pycompat.xrange(len(a)):
             if a[i] != b[i]:
                 return a[:i]
         return a
 
     try:
         if not filelist:
             return b""
         dirlist = [f.lstrip(b'/').split(b'/')[:-1] for f in filelist]
         if len(dirlist) == 1:
             return b'/'.join(dirlist[0])
         a = min(dirlist)
         b = max(dirlist)
         # The common prefix of a and b is shared with all
         # elements of the list since Python sorts lexicographical
         # and [1, x] after [1].
         return b'/'.join(common(a, b))
     except TypeError:
         raise error.ParseError(_(b'argument is not a list of text'))
 
 
 @templatefilter(b'count')
 def count(i):
     """List or text. Returns the length as an integer."""
     try:
         return len(i)
     except TypeError:
         raise error.ParseError(_(b'not countable'))
 
 
 @templatefilter(b'dirname', intype=bytes)
 def dirname(path):
     """Any text. Treats the text as a path, and strips the last
     component of the path after splitting by the path separator.
     """
     return os.path.dirname(path)
 
 
 @templatefilter(b'domain', intype=bytes)
 def domain(author):
     """Any text. Finds the first string that looks like an email
     address, and extracts just the domain component. Example: ``User
     <[e-mail]>`` becomes ``example.com``.
     """
     f = author.find(b'@')
     if f == -1:
         return b''
     author = author[f + 1 :]
     f = author.find(b'>')
     if f >= 0:
         author = author[:f]
     return author
 
 
 @templatefilter(b'email', intype=bytes)
 def email(text):
     """Any text. Extracts the first string that looks like an email
     address. Example: ``User <[e-mail]>`` becomes
     ``[e-mail]``.
     """
     return stringutil.email(text)
 
 
 @templatefilter(b'escape', intype=bytes)
 def escape(text):
     """Any text. Replaces the special XML/XHTML characters "&", "<"
     and ">" with XML entities, and filters out NUL characters.
     """
     return url.escape(text.replace(b'\0', b''), True)
 
 
 para_re = None
 space_re = None
 
 
 def fill(text, width, initindent=b'', hangindent=b''):
     '''fill many paragraphs with optional indentation.'''
     global para_re, space_re
     if para_re is None:
         para_re = re.compile(b'(\n\n|\n\\s*[-*]\\s*)', re.M)
         space_re = re.compile(br'  +')
 
     def findparas():
         start = 0
         while True:
             m = para_re.search(text, start)
             if not m:
                 uctext = encoding.unifromlocal(text[start:])
                 w = len(uctext)
                 while w > 0 and uctext[w - 1].isspace():
                     w -= 1
                 yield (
                     encoding.unitolocal(uctext[:w]),
                     encoding.unitolocal(uctext[w:]),
                 )
                 break
             yield text[start : m.start(0)], m.group(1)
             start = m.end(1)
 
     return b"".join(
         [
             stringutil.wrap(
                 space_re.sub(b' ', stringutil.wrap(para, width)),
                 width,
                 initindent,
                 hangindent,
             )
             + rest
             for para, rest in findparas()
         ]
     )
 
 
 @templatefilter(b'fill68', intype=bytes)
 def fill68(text):
     """Any text. Wraps the text to fit in 68 columns."""
     return fill(text, 68)
 
 
 @templatefilter(b'fill76', intype=bytes)
 def fill76(text):
     """Any text. Wraps the text to fit in 76 columns."""
     return fill(text, 76)
 
 
 @templatefilter(b'firstline', intype=bytes)
 def firstline(text):
     """Any text. Returns the first line of text."""
     try:
         return text.splitlines(True)[0].rstrip(b'\r\n')
     except IndexError:
         return b''
 
 
 @templatefilter(b'hex', intype=bytes)
 def hexfilter(text):
     """Any text. Convert a binary Mercurial node identifier into
     its long hexadecimal representation.
     """
     return hex(text)
 
 
 @templatefilter(b'hgdate', intype=templateutil.date)
 def hgdate(text):
     """Date. Returns the date as a pair of numbers: "1157407993
     25200" (Unix timestamp, timezone offset).
     """
     return b"%d %d" % text
 
 
 @templatefilter(b'isodate', intype=templateutil.date)
 def isodate(text):
     """Date. Returns the date in ISO 8601 format: "2009-08-18 13:00
     +0200".
     """
     return dateutil.datestr(text, b'%Y-%m-%d %H:%M %1%2')
 
 
 @templatefilter(b'isodatesec', intype=templateutil.date)
 def isodatesec(text):
     """Date. Returns the date in ISO 8601 format, including
     seconds: "2009-08-18 13:00:13 +0200". See also the rfc3339date
     filter.
     """
     return dateutil.datestr(text, b'%Y-%m-%d %H:%M:%S %1%2')
 
 
 def indent(text, prefix, firstline=b''):
     '''indent each non-empty line of text after first with prefix.'''
     lines = text.splitlines()
     num_lines = len(lines)
     endswithnewline = text[-1:] == b'\n'
 
     def indenter():
         for i in pycompat.xrange(num_lines):
             l = lines[i]
             if l.strip():
                 yield prefix if i else firstline
             yield l
             if i < num_lines - 1 or endswithnewline:
                 yield b'\n'
 
     return b"".join(indenter())
 
 
 @templatefilter(b'json')
 def json(obj, paranoid=True):
     """Any object. Serializes the object to a JSON formatted text."""
     if obj is None:
         return b'null'
     elif obj is False:
         return b'false'
     elif obj is True:
         return b'true'
-    elif isinstance(obj, (int, pycompat.long, float)):
+    elif isinstance(obj, (int, int, float)):
         return pycompat.bytestr(obj)
     elif isinstance(obj, bytes):
         return b'"%s"' % encoding.jsonescape(obj, paranoid=paranoid)
     elif isinstance(obj, type(u'')):
         raise error.ProgrammingError(
             b'Mercurial only does output with bytes: %r' % obj
         )
     elif util.safehasattr(obj, b'keys'):
         out = [
             b'"%s": %s'
             % (encoding.jsonescape(k, paranoid=paranoid), json(v, paranoid))
             for k, v in sorted(obj.items())
         ]
         return b'{' + b', '.join(out) + b'}'
     elif util.safehasattr(obj, b'__iter__'):
         out = [json(i, paranoid) for i in obj]
         return b'[' + b', '.join(out) + b']'
     raise error.ProgrammingError(b'cannot encode %r' % obj)
 
 
 @templatefilter(b'lower', intype=bytes)
 def lower(text):
     """Any text. Converts the text to lowercase."""
     return encoding.lower(text)
 
 
 @templatefilter(b'nonempty', intype=bytes)
 def nonempty(text):
     """Any text. Returns '(none)' if the string is empty."""
     return text or b"(none)"
 
 
 @templatefilter(b'obfuscate', intype=bytes)
 def obfuscate(text):
     """Any text. Returns the input text rendered as a sequence of
     XML entities.
     """
     text = pycompat.unicode(
         text, pycompat.sysstr(encoding.encoding), r'replace'
     )
     return b''.join([b'&#%d;' % ord(c) for c in text])
 
 
 @templatefilter(b'permissions', intype=bytes)
 def permissions(flags):
     if b"l" in flags:
         return b"lrwxrwxrwx"
     if b"x" in flags:
         return b"-rwxr-xr-x"
     return b"-rw-r--r--"
 
 
 @templatefilter(b'person', intype=bytes)
 def person(author):
     """Any text. Returns the name before an email address,
     interpreting it as per RFC 5322.
     """
     return stringutil.person(author)
 
 
 @templatefilter(b'revescape', intype=bytes)
 def revescape(text):
     """Any text. Escapes all "special" characters, except @.
     Forward slashes are escaped twice to prevent web servers from prematurely
     unescaping them. For example, "@foo bar/baz" becomes "@foo%20bar%252Fbaz".
     """
     return urlreq.quote(text, safe=b'/@').replace(b'/', b'%252F')
 
 
 @templatefilter(b'rfc3339date', intype=templateutil.date)
 def rfc3339date(text):
     """Date. Returns a date using the Internet date format
     specified in RFC 3339: "2009-08-18T13:00:13+02:00".
     """
     return dateutil.datestr(text, b"%Y-%m-%dT%H:%M:%S%1:%2")
 
 
 @templatefilter(b'rfc822date', intype=templateutil.date)
 def rfc822date(text):
     """Date. Returns a date using the same format used in email
     headers: "Tue, 18 Aug 2009 13:00:13 +0200".
     """
     return dateutil.datestr(text, b"%a, %d %b %Y %H:%M:%S %1%2")
 
 
 @templatefilter(b'short', intype=bytes)
 def short(text):
     """Changeset hash. Returns the short form of a changeset hash,
     i.e. a 12 hexadecimal digit string.
     """
     return text[:12]
 
 
 @templatefilter(b'shortbisect', intype=bytes)
 def shortbisect(label):
     """Any text. Treats `label` as a bisection status, and
     returns a single-character representing the status (G: good, B: bad,
     S: skipped, U: untested, I: ignored). Returns single space if `text`
     is not a valid bisection status.
     """
     if label:
         return label[0:1].upper()
     return b' '
 
 
 @templatefilter(b'shortdate', intype=templateutil.date)
 def shortdate(text):
     """Date. Returns a date like "2006-09-18"."""
     return dateutil.shortdate(text)
 
 
 @templatefilter(b'slashpath', intype=bytes)
 def slashpath(path):
     """Any text. Replaces the native path separator with slash."""
     return util.pconvert(path)
 
 
 @templatefilter(b'splitlines', intype=bytes)
 def splitlines(text):
     """Any text. Split text into a list of lines."""
     return templateutil.hybridlist(text.splitlines(), name=b'line')
 
 
 @templatefilter(b'stringescape', intype=bytes)
 def stringescape(text):
     return stringutil.escapestr(text)
 
 
 @templatefilter(b'stringify', intype=bytes)
 def stringify(thing):
     """Any type. Turns the value into text by converting values into
     text and concatenating them.
     """
     return thing  # coerced by the intype
 
 
 @templatefilter(b'stripdir', intype=bytes)
 def stripdir(text):
     """Treat the text as path and strip a directory level, if
     possible. For example, "foo" and "foo/bar" becomes "foo".
     """
     dir = os.path.dirname(text)
     if dir == b"":
         return os.path.basename(text)
     else:
         return dir
 
 
 @templatefilter(b'tabindent', intype=bytes)
 def tabindent(text):
     """Any text. Returns the text, with every non-empty line
     except the first starting with a tab character.
     """
     return indent(text, b'\t')
 
 
 @templatefilter(b'upper', intype=bytes)
 def upper(text):
     """Any text. Converts the text to uppercase."""
     return encoding.upper(text)
 
 
 @templatefilter(b'urlescape', intype=bytes)
 def urlescape(text):
     """Any text. Escapes all "special" characters. For example,
     "foo bar" becomes "foo%20bar".
     """
     return urlreq.quote(text)
 
 
 @templatefilter(b'user', intype=bytes)
 def userfilter(text):
     """Any text. Returns a short representation of a user name or email
     address."""
     return stringutil.shortuser(text)
 
 
 @templatefilter(b'emailuser', intype=bytes)
 def emailuser(text):
     """Any text. Returns the user portion of an email address."""
     return stringutil.emailuser(text)
 
 
 @templatefilter(b'utf8', intype=bytes)
 def utf8(text):
     """Any text. Converts from the local character encoding to UTF-8."""
     return encoding.fromlocal(text)
 
 
 @templatefilter(b'xmlescape', intype=bytes)
 def xmlescape(text):
     text = (
         text.replace(b'&', b'&amp;')
         .replace(b'<', b'&lt;')
         .replace(b'>', b'&gt;')
         .replace(b'"', b'&quot;')
         .replace(b"'", b'&#39;')
     )  # &apos; invalid in HTML
     return re.sub(b'[\x00-\x08\x0B\x0C\x0E-\x1F]', b' ', text)
 
 
 def websub(text, websubtable):
     """:websub: Any text. Only applies to hgweb. Applies the regular
     expression replacements defined in the websub section.
     """
     if websubtable:
         for regexp, format in websubtable:
             text = regexp.sub(format, text)
     return text
 
 
 def loadfilter(ui, extname, registrarobj):
     """Load template filter from specified registrarobj"""
     for name, func in registrarobj._table.items():
         filters[name] = func
 
 
 # tell hggettext to extract docstrings from these functions:
 i18nfunctions = filters.values()

mercurial/utils/cborutil.py

@@ -1,1082 +1,1081 b''
 # cborutil.py - CBOR extensions
 #
 # Copyright 2018 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 
 import struct
 import sys
 
-from .. import pycompat
 
 # Very short very of RFC 7049...
 #
 # Each item begins with a byte. The 3 high bits of that byte denote the
 # "major type." The lower 5 bits denote the "subtype." Each major type
 # has its own encoding mechanism.
 #
 # Most types have lengths. However, bytestring, string, array, and map
 # can be indefinite length. These are denotes by a subtype with value 31.
 # Sub-components of those types then come afterwards and are terminated
 # by a "break" byte.
 
 MAJOR_TYPE_UINT = 0
 MAJOR_TYPE_NEGINT = 1
 MAJOR_TYPE_BYTESTRING = 2
 MAJOR_TYPE_STRING = 3
 MAJOR_TYPE_ARRAY = 4
 MAJOR_TYPE_MAP = 5
 MAJOR_TYPE_SEMANTIC = 6
 MAJOR_TYPE_SPECIAL = 7
 
 SUBTYPE_MASK = 0b00011111
 
 SUBTYPE_FALSE = 20
 SUBTYPE_TRUE = 21
 SUBTYPE_NULL = 22
 SUBTYPE_HALF_FLOAT = 25
 SUBTYPE_SINGLE_FLOAT = 26
 SUBTYPE_DOUBLE_FLOAT = 27
 SUBTYPE_INDEFINITE = 31
 
 SEMANTIC_TAG_FINITE_SET = 258
 
 # Indefinite types begin with their major type ORd with information value 31.
 BEGIN_INDEFINITE_BYTESTRING = struct.pack(
     '>B', MAJOR_TYPE_BYTESTRING << 5 | SUBTYPE_INDEFINITE
 )
 BEGIN_INDEFINITE_ARRAY = struct.pack(
     '>B', MAJOR_TYPE_ARRAY << 5 | SUBTYPE_INDEFINITE
 )
 BEGIN_INDEFINITE_MAP = struct.pack(
     '>B', MAJOR_TYPE_MAP << 5 | SUBTYPE_INDEFINITE
 )
 
 ENCODED_LENGTH_1 = struct.Struct('>B')
 ENCODED_LENGTH_2 = struct.Struct('>BB')
 ENCODED_LENGTH_3 = struct.Struct('>BH')
 ENCODED_LENGTH_4 = struct.Struct('>BL')
 ENCODED_LENGTH_5 = struct.Struct('>BQ')
 
 # The break ends an indefinite length item.
 BREAK = b'\xff'
 BREAK_INT = 255
 
 
 def encodelength(majortype, length):
     """Obtain a value encoding the major type and its length."""
     if length < 24:
         return ENCODED_LENGTH_1.pack(majortype << 5 | length)
     elif length < 256:
         return ENCODED_LENGTH_2.pack(majortype << 5 | 24, length)
     elif length < 65536:
         return ENCODED_LENGTH_3.pack(majortype << 5 | 25, length)
     elif length < 4294967296:
         return ENCODED_LENGTH_4.pack(majortype << 5 | 26, length)
     else:
         return ENCODED_LENGTH_5.pack(majortype << 5 | 27, length)
 
 
 def streamencodebytestring(v):
     yield encodelength(MAJOR_TYPE_BYTESTRING, len(v))
     yield v
 
 
 def streamencodebytestringfromiter(it):
     """Convert an iterator of chunks to an indefinite bytestring.
 
     Given an input that is iterable and each element in the iterator is
     representable as bytes, emit an indefinite length bytestring.
     """
     yield BEGIN_INDEFINITE_BYTESTRING
 
     for chunk in it:
         yield encodelength(MAJOR_TYPE_BYTESTRING, len(chunk))
         yield chunk
 
     yield BREAK
 
 
 def streamencodeindefinitebytestring(source, chunksize=65536):
     """Given a large source buffer, emit as an indefinite length bytestring.
 
     This is a generator of chunks constituting the encoded CBOR data.
     """
     yield BEGIN_INDEFINITE_BYTESTRING
 
     i = 0
     l = len(source)
 
     while True:
         chunk = source[i : i + chunksize]
         i += len(chunk)
 
         yield encodelength(MAJOR_TYPE_BYTESTRING, len(chunk))
         yield chunk
 
         if i >= l:
             break
 
     yield BREAK
 
 
 def streamencodeint(v):
     if v >= 18446744073709551616 or v < -18446744073709551616:
         raise ValueError(b'big integers not supported')
 
     if v >= 0:
         yield encodelength(MAJOR_TYPE_UINT, v)
     else:
         yield encodelength(MAJOR_TYPE_NEGINT, abs(v) - 1)
 
 
 def streamencodearray(l):
     """Encode a known size iterable to an array."""
 
     yield encodelength(MAJOR_TYPE_ARRAY, len(l))
 
     for i in l:
         for chunk in streamencode(i):
             yield chunk
 
 
 def streamencodearrayfromiter(it):
     """Encode an iterator of items to an indefinite length array."""
 
     yield BEGIN_INDEFINITE_ARRAY
 
     for i in it:
         for chunk in streamencode(i):
             yield chunk
 
     yield BREAK
 
 
 def _mixedtypesortkey(v):
     return type(v).__name__, v
 
 
 def streamencodeset(s):
     # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml defines
     # semantic tag 258 for finite sets.
     yield encodelength(MAJOR_TYPE_SEMANTIC, SEMANTIC_TAG_FINITE_SET)
 
     for chunk in streamencodearray(sorted(s, key=_mixedtypesortkey)):
         yield chunk
 
 
 def streamencodemap(d):
     """Encode dictionary to a generator.
 
     Does not supporting indefinite length dictionaries.
     """
     yield encodelength(MAJOR_TYPE_MAP, len(d))
 
     for key, value in sorted(d.items(), key=lambda x: _mixedtypesortkey(x[0])):
         for chunk in streamencode(key):
             yield chunk
         for chunk in streamencode(value):
             yield chunk
 
 
 def streamencodemapfromiter(it):
     """Given an iterable of (key, value), encode to an indefinite length map."""
     yield BEGIN_INDEFINITE_MAP
 
     for key, value in it:
         for chunk in streamencode(key):
             yield chunk
         for chunk in streamencode(value):
             yield chunk
 
     yield BREAK
 
 
 def streamencodebool(b):
     # major type 7, simple value 20 and 21.
     yield b'\xf5' if b else b'\xf4'
 
 
 def streamencodenone(v):
     # major type 7, simple value 22.
     yield b'\xf6'
 
 
 STREAM_ENCODERS = {
     bytes: streamencodebytestring,
     int: streamencodeint,
-    pycompat.long: streamencodeint,
+    int: streamencodeint,
     list: streamencodearray,
     tuple: streamencodearray,
     dict: streamencodemap,
     set: streamencodeset,
     bool: streamencodebool,
     type(None): streamencodenone,
 }
 
 
 def streamencode(v):
     """Encode a value in a streaming manner.
 
     Given an input object, encode it to CBOR recursively.
 
     Returns a generator of CBOR encoded bytes. There is no guarantee
     that each emitted chunk fully decodes to a value or sub-value.
 
     Encoding is deterministic - unordered collections are sorted.
     """
     fn = STREAM_ENCODERS.get(v.__class__)
 
     if not fn:
         # handle subtypes such as encoding.localstr and util.sortdict
         for ty in STREAM_ENCODERS:
             if not isinstance(v, ty):
                 continue
             fn = STREAM_ENCODERS[ty]
             break
 
     if not fn:
         raise ValueError(b'do not know how to encode %s' % type(v))
 
     return fn(v)
 
 
 class CBORDecodeError(Exception):
     """Represents an error decoding CBOR."""
 
 
 if sys.version_info.major >= 3:
 
     def _elementtointeger(b, i):
         return b[i]
 
 
 else:
 
     def _elementtointeger(b, i):
         return ord(b[i])
 
 
 STRUCT_BIG_UBYTE = struct.Struct('>B')
 STRUCT_BIG_USHORT = struct.Struct(b'>H')
 STRUCT_BIG_ULONG = struct.Struct(b'>L')
 STRUCT_BIG_ULONGLONG = struct.Struct(b'>Q')
 
 SPECIAL_NONE = 0
 SPECIAL_START_INDEFINITE_BYTESTRING = 1
 SPECIAL_START_ARRAY = 2
 SPECIAL_START_MAP = 3
 SPECIAL_START_SET = 4
 SPECIAL_INDEFINITE_BREAK = 5
 
 
 def decodeitem(b, offset=0):
     """Decode a new CBOR value from a buffer at offset.
 
     This function attempts to decode up to one complete CBOR value
     from ``b`` starting at offset ``offset``.
 
     The beginning of a collection (such as an array, map, set, or
     indefinite length bytestring) counts as a single value. For these
     special cases, a state flag will indicate that a special value was seen.
 
     When called, the function either returns a decoded value or gives
     a hint as to how many more bytes are needed to do so. By calling
     the function repeatedly given a stream of bytes, the caller can
     build up the original values.
 
     Returns a tuple with the following elements:
 
     * Bool indicating whether a complete value was decoded.
     * A decoded value if first value is True otherwise None
     * Integer number of bytes. If positive, the number of bytes
       read. If negative, the number of bytes we need to read to
       decode this value or the next chunk in this value.
     * One of the ``SPECIAL_*`` constants indicating special treatment
       for this value. ``SPECIAL_NONE`` means this is a fully decoded
       simple value (such as an integer or bool).
     """
 
     initial = _elementtointeger(b, offset)
     offset += 1
 
     majortype = initial >> 5
     subtype = initial & SUBTYPE_MASK
 
     if majortype == MAJOR_TYPE_UINT:
         complete, value, readcount = decodeuint(subtype, b, offset)
 
         if complete:
             return True, value, readcount + 1, SPECIAL_NONE
         else:
             return False, None, readcount, SPECIAL_NONE
 
     elif majortype == MAJOR_TYPE_NEGINT:
         # Negative integers are the same as UINT except inverted minus 1.
         complete, value, readcount = decodeuint(subtype, b, offset)
 
         if complete:
             return True, -value - 1, readcount + 1, SPECIAL_NONE
         else:
             return False, None, readcount, SPECIAL_NONE
 
     elif majortype == MAJOR_TYPE_BYTESTRING:
         # Beginning of bytestrings are treated as uints in order to
         # decode their length, which may be indefinite.
         complete, size, readcount = decodeuint(
             subtype, b, offset, allowindefinite=True
         )
 
         # We don't know the size of the bytestring. It must be a definitive
         # length since the indefinite subtype would be encoded in the initial
         # byte.
         if not complete:
             return False, None, readcount, SPECIAL_NONE
 
         # We know the length of the bytestring.
         if size is not None:
             # And the data is available in the buffer.
             if offset + readcount + size <= len(b):
                 value = b[offset + readcount : offset + readcount + size]
                 return True, value, readcount + size + 1, SPECIAL_NONE
 
             # And we need more data in order to return the bytestring.
             else:
                 wanted = len(b) - offset - readcount - size
                 return False, None, wanted, SPECIAL_NONE
 
         # It is an indefinite length bytestring.
         else:
             return True, None, 1, SPECIAL_START_INDEFINITE_BYTESTRING
 
     elif majortype == MAJOR_TYPE_STRING:
         raise CBORDecodeError(b'string major type not supported')
 
     elif majortype == MAJOR_TYPE_ARRAY:
         # Beginning of arrays are treated as uints in order to decode their
         # length. We don't allow indefinite length arrays.
         complete, size, readcount = decodeuint(subtype, b, offset)
 
         if complete:
             return True, size, readcount + 1, SPECIAL_START_ARRAY
         else:
             return False, None, readcount, SPECIAL_NONE
 
     elif majortype == MAJOR_TYPE_MAP:
         # Beginning of maps are treated as uints in order to decode their
         # number of elements. We don't allow indefinite length arrays.
         complete, size, readcount = decodeuint(subtype, b, offset)
 
         if complete:
             return True, size, readcount + 1, SPECIAL_START_MAP
         else:
             return False, None, readcount, SPECIAL_NONE
 
     elif majortype == MAJOR_TYPE_SEMANTIC:
         # Semantic tag value is read the same as a uint.
         complete, tagvalue, readcount = decodeuint(subtype, b, offset)
 
         if not complete:
             return False, None, readcount, SPECIAL_NONE
 
         # This behavior here is a little wonky. The main type being "decorated"
         # by this semantic tag follows. A more robust parser would probably emit
         # a special flag indicating this as a semantic tag and let the caller
         # deal with the types that follow. But since we don't support many
         # semantic tags, it is easier to deal with the special cases here and
         # hide complexity from the caller. If we add support for more semantic
         # tags, we should probably move semantic tag handling into the caller.
         if tagvalue == SEMANTIC_TAG_FINITE_SET:
             if offset + readcount >= len(b):
                 return False, None, -1, SPECIAL_NONE
 
             complete, size, readcount2, special = decodeitem(
                 b, offset + readcount
             )
 
             if not complete:
                 return False, None, readcount2, SPECIAL_NONE
 
             if special != SPECIAL_START_ARRAY:
                 raise CBORDecodeError(
                     b'expected array after finite set semantic tag'
                 )
 
             return True, size, readcount + readcount2 + 1, SPECIAL_START_SET
 
         else:
             raise CBORDecodeError(b'semantic tag %d not allowed' % tagvalue)
 
     elif majortype == MAJOR_TYPE_SPECIAL:
         # Only specific values for the information field are allowed.
         if subtype == SUBTYPE_FALSE:
             return True, False, 1, SPECIAL_NONE
         elif subtype == SUBTYPE_TRUE:
             return True, True, 1, SPECIAL_NONE
         elif subtype == SUBTYPE_NULL:
             return True, None, 1, SPECIAL_NONE
         elif subtype == SUBTYPE_INDEFINITE:
             return True, None, 1, SPECIAL_INDEFINITE_BREAK
         # If value is 24, subtype is in next byte.
         else:
             raise CBORDecodeError(b'special type %d not allowed' % subtype)
     else:
         assert False
 
 
 def decodeuint(subtype, b, offset=0, allowindefinite=False):
     """Decode an unsigned integer.
 
     ``subtype`` is the lower 5 bits from the initial byte CBOR item
     "header." ``b`` is a buffer containing bytes. ``offset`` points to
     the index of the first byte after the byte that ``subtype`` was
     derived from.
 
     ``allowindefinite`` allows the special indefinite length value
     indicator.
 
     Returns a 3-tuple of (successful, value, count).
 
     The first element is a bool indicating if decoding completed. The 2nd
     is the decoded integer value or None if not fully decoded or the subtype
     is 31 and ``allowindefinite`` is True. The 3rd value is the count of bytes.
     If positive, it is the number of additional bytes decoded. If negative,
     it is the number of additional bytes needed to decode this value.
     """
 
     # Small values are inline.
     if subtype < 24:
         return True, subtype, 0
     # Indefinite length specifier.
     elif subtype == 31:
         if allowindefinite:
             return True, None, 0
         else:
             raise CBORDecodeError(b'indefinite length uint not allowed here')
     elif subtype >= 28:
         raise CBORDecodeError(
             b'unsupported subtype on integer type: %d' % subtype
         )
 
     if subtype == 24:
         s = STRUCT_BIG_UBYTE
     elif subtype == 25:
         s = STRUCT_BIG_USHORT
     elif subtype == 26:
         s = STRUCT_BIG_ULONG
     elif subtype == 27:
         s = STRUCT_BIG_ULONGLONG
     else:
         raise CBORDecodeError(b'bounds condition checking violation')
 
     if len(b) - offset >= s.size:
         return True, s.unpack_from(b, offset)[0], s.size
     else:
         return False, None, len(b) - offset - s.size
 
 
 class bytestringchunk(bytes):
     """Represents a chunk/segment in an indefinite length bytestring.
 
     This behaves like a ``bytes`` but in addition has the ``isfirst``
     and ``islast`` attributes indicating whether this chunk is the first
     or last in an indefinite length bytestring.
     """
 
     def __new__(cls, v, first=False, last=False):
         self = bytes.__new__(cls, v)
         self.isfirst = first
         self.islast = last
 
         return self
 
 
 class sansiodecoder(object):
     """A CBOR decoder that doesn't perform its own I/O.
 
     To use, construct an instance and feed it segments containing
     CBOR-encoded bytes via ``decode()``. The return value from ``decode()``
     indicates whether a fully-decoded value is available, how many bytes
     were consumed, and offers a hint as to how many bytes should be fed
     in next time to decode the next value.
 
     The decoder assumes it will decode N discrete CBOR values, not just
     a single value. i.e. if the bytestream contains uints packed one after
     the other, the decoder will decode them all, rather than just the initial
     one.
 
     When ``decode()`` indicates a value is available, call ``getavailable()``
     to return all fully decoded values.
 
     ``decode()`` can partially decode input. It is up to the caller to keep
     track of what data was consumed and to pass unconsumed data in on the
     next invocation.
 
     The decoder decodes atomically at the *item* level. See ``decodeitem()``.
     If an *item* cannot be fully decoded, the decoder won't record it as
     partially consumed. Instead, the caller will be instructed to pass in
     the initial bytes of this item on the next invocation. This does result
     in some redundant parsing. But the overhead should be minimal.
 
     This decoder only supports a subset of CBOR as required by Mercurial.
     It lacks support for:
 
     * Indefinite length arrays
     * Indefinite length maps
     * Use of indefinite length bytestrings as keys or values within
       arrays, maps, or sets.
     * Nested arrays, maps, or sets within sets
     * Any semantic tag that isn't a mathematical finite set
     * Floating point numbers
     * Undefined special value
 
     CBOR types are decoded to Python types as follows:
 
     uint -> int
     negint -> int
     bytestring -> bytes
     map -> dict
     array -> list
     True -> bool
     False -> bool
     null -> None
     indefinite length bytestring chunk -> [bytestringchunk]
 
     The only non-obvious mapping here is an indefinite length bytestring
     to the ``bytestringchunk`` type. This is to facilitate streaming
     indefinite length bytestrings out of the decoder and to differentiate
     a regular bytestring from an indefinite length bytestring.
     """
 
     _STATE_NONE = 0
     _STATE_WANT_MAP_KEY = 1
     _STATE_WANT_MAP_VALUE = 2
     _STATE_WANT_ARRAY_VALUE = 3
     _STATE_WANT_SET_VALUE = 4
     _STATE_WANT_BYTESTRING_CHUNK_FIRST = 5
     _STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT = 6
 
     def __init__(self):
         # TODO add support for limiting size of bytestrings
         # TODO add support for limiting number of keys / values in collections
         # TODO add support for limiting size of buffered partial values
 
         self.decodedbytecount = 0
 
         self._state = self._STATE_NONE
 
         # Stack of active nested collections. Each entry is a dict describing
         # the collection.
         self._collectionstack = []
 
         # Fully decoded key to use for the current map.
         self._currentmapkey = None
 
         # Fully decoded values available for retrieval.
         self._decodedvalues = []
 
     @property
     def inprogress(self):
         """Whether the decoder has partially decoded a value."""
         return self._state != self._STATE_NONE
 
     def decode(self, b, offset=0):
         """Attempt to decode bytes from an input buffer.
 
         ``b`` is a collection of bytes and ``offset`` is the byte
         offset within that buffer from which to begin reading data.
 
         ``b`` must support ``len()`` and accessing bytes slices via
         ``__slice__``. Typically ``bytes`` instances are used.
 
         Returns a tuple with the following fields:
 
         * Bool indicating whether values are available for retrieval.
         * Integer indicating the number of bytes that were fully consumed,
           starting from ``offset``.
         * Integer indicating the number of bytes that are desired for the
           next call in order to decode an item.
         """
         if not b:
             return bool(self._decodedvalues), 0, 0
 
         initialoffset = offset
 
         # We could easily split the body of this loop into a function. But
         # Python performance is sensitive to function calls and collections
         # are composed of many items. So leaving as a while loop could help
         # with performance. One thing that may not help is the use of
         # if..elif versus a lookup/dispatch table. There may be value
         # in switching that.
         while offset < len(b):
             # Attempt to decode an item. This could be a whole value or a
             # special value indicating an event, such as start or end of a
             # collection or indefinite length type.
             complete, value, readcount, special = decodeitem(b, offset)
 
             if readcount > 0:
                 self.decodedbytecount += readcount
 
             if not complete:
                 assert readcount < 0
                 return (
                     bool(self._decodedvalues),
                     offset - initialoffset,
                     -readcount,
                 )
 
             offset += readcount
 
             # No nested state. We either have a full value or beginning of a
             # complex value to deal with.
             if self._state == self._STATE_NONE:
                 # A normal value.
                 if special == SPECIAL_NONE:
                     self._decodedvalues.append(value)
 
                 elif special == SPECIAL_START_ARRAY:
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': [],
                         }
                     )
                     self._state = self._STATE_WANT_ARRAY_VALUE
 
                 elif special == SPECIAL_START_MAP:
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': {},
                         }
                     )
                     self._state = self._STATE_WANT_MAP_KEY
 
                 elif special == SPECIAL_START_SET:
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': set(),
                         }
                     )
                     self._state = self._STATE_WANT_SET_VALUE
 
                 elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
                     self._state = self._STATE_WANT_BYTESTRING_CHUNK_FIRST
 
                 else:
                     raise CBORDecodeError(
                         b'unhandled special state: %d' % special
                     )
 
             # This value becomes an element of the current array.
             elif self._state == self._STATE_WANT_ARRAY_VALUE:
                 # Simple values get appended.
                 if special == SPECIAL_NONE:
                     c = self._collectionstack[-1]
                     c[b'v'].append(value)
                     c[b'remaining'] -= 1
 
                     # self._state doesn't need changed.
 
                 # An array nested within an array.
                 elif special == SPECIAL_START_ARRAY:
                     lastc = self._collectionstack[-1]
                     newvalue = []
 
                     lastc[b'v'].append(newvalue)
                     lastc[b'remaining'] -= 1
 
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': newvalue,
                         }
                     )
 
                     # self._state doesn't need changed.
 
                 # A map nested within an array.
                 elif special == SPECIAL_START_MAP:
                     lastc = self._collectionstack[-1]
                     newvalue = {}
 
                     lastc[b'v'].append(newvalue)
                     lastc[b'remaining'] -= 1
 
                     self._collectionstack.append(
                         {b'remaining': value, b'v': newvalue}
                     )
 
                     self._state = self._STATE_WANT_MAP_KEY
 
                 elif special == SPECIAL_START_SET:
                     lastc = self._collectionstack[-1]
                     newvalue = set()
 
                     lastc[b'v'].append(newvalue)
                     lastc[b'remaining'] -= 1
 
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': newvalue,
                         }
                     )
 
                     self._state = self._STATE_WANT_SET_VALUE
 
                 elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
                     raise CBORDecodeError(
                         b'indefinite length bytestrings '
                         b'not allowed as array values'
                     )
 
                 else:
                     raise CBORDecodeError(
                         b'unhandled special item when '
                         b'expecting array value: %d' % special
                     )
 
             # This value becomes the key of the current map instance.
             elif self._state == self._STATE_WANT_MAP_KEY:
                 if special == SPECIAL_NONE:
                     self._currentmapkey = value
                     self._state = self._STATE_WANT_MAP_VALUE
 
                 elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
                     raise CBORDecodeError(
                         b'indefinite length bytestrings '
                         b'not allowed as map keys'
                     )
 
                 elif special in (
                     SPECIAL_START_ARRAY,
                     SPECIAL_START_MAP,
                     SPECIAL_START_SET,
                 ):
                     raise CBORDecodeError(
                         b'collections not supported as map keys'
                     )
 
                 # We do not allow special values to be used as map keys.
                 else:
                     raise CBORDecodeError(
                         b'unhandled special item when '
                         b'expecting map key: %d' % special
                     )
 
             # This value becomes the value of the current map key.
             elif self._state == self._STATE_WANT_MAP_VALUE:
                 # Simple values simply get inserted into the map.
                 if special == SPECIAL_NONE:
                     lastc = self._collectionstack[-1]
                     lastc[b'v'][self._currentmapkey] = value
                     lastc[b'remaining'] -= 1
 
                     self._state = self._STATE_WANT_MAP_KEY
 
                 # A new array is used as the map value.
                 elif special == SPECIAL_START_ARRAY:
                     lastc = self._collectionstack[-1]
                     newvalue = []
 
                     lastc[b'v'][self._currentmapkey] = newvalue
                     lastc[b'remaining'] -= 1
 
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': newvalue,
                         }
                     )
 
                     self._state = self._STATE_WANT_ARRAY_VALUE
 
                 # A new map is used as the map value.
                 elif special == SPECIAL_START_MAP:
                     lastc = self._collectionstack[-1]
                     newvalue = {}
 
                     lastc[b'v'][self._currentmapkey] = newvalue
                     lastc[b'remaining'] -= 1
 
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': newvalue,
                         }
                     )
 
                     self._state = self._STATE_WANT_MAP_KEY
 
                 # A new set is used as the map value.
                 elif special == SPECIAL_START_SET:
                     lastc = self._collectionstack[-1]
                     newvalue = set()
 
                     lastc[b'v'][self._currentmapkey] = newvalue
                     lastc[b'remaining'] -= 1
 
                     self._collectionstack.append(
                         {
                             b'remaining': value,
                             b'v': newvalue,
                         }
                     )
 
                     self._state = self._STATE_WANT_SET_VALUE
 
                 elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
                     raise CBORDecodeError(
                         b'indefinite length bytestrings not '
                         b'allowed as map values'
                     )
 
                 else:
                     raise CBORDecodeError(
                         b'unhandled special item when '
                         b'expecting map value: %d' % special
                     )
 
                 self._currentmapkey = None
 
             # This value is added to the current set.
             elif self._state == self._STATE_WANT_SET_VALUE:
                 if special == SPECIAL_NONE:
                     lastc = self._collectionstack[-1]
                     lastc[b'v'].add(value)
                     lastc[b'remaining'] -= 1
 
                 elif special == SPECIAL_START_INDEFINITE_BYTESTRING:
                     raise CBORDecodeError(
                         b'indefinite length bytestrings not '
                         b'allowed as set values'
                     )
 
                 elif special in (
                     SPECIAL_START_ARRAY,
                     SPECIAL_START_MAP,
                     SPECIAL_START_SET,
                 ):
                     raise CBORDecodeError(
                         b'collections not allowed as set values'
                     )
 
                 # We don't allow non-trivial types to exist as set values.
                 else:
                     raise CBORDecodeError(
                         b'unhandled special item when '
                         b'expecting set value: %d' % special
                     )
 
             # This value represents the first chunk in an indefinite length
             # bytestring.
             elif self._state == self._STATE_WANT_BYTESTRING_CHUNK_FIRST:
                 # We received a full chunk.
                 if special == SPECIAL_NONE:
                     self._decodedvalues.append(
                         bytestringchunk(value, first=True)
                     )
 
                     self._state = self._STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT
 
                 # The end of stream marker. This means it is an empty
                 # indefinite length bytestring.
                 elif special == SPECIAL_INDEFINITE_BREAK:
                     # We /could/ convert this to a b''. But we want to preserve
                     # the nature of the underlying data so consumers expecting
                     # an indefinite length bytestring get one.
                     self._decodedvalues.append(
                         bytestringchunk(b'', first=True, last=True)
                     )
 
                     # Since indefinite length bytestrings can't be used in
                     # collections, we must be at the root level.
                     assert not self._collectionstack
                     self._state = self._STATE_NONE
 
                 else:
                     raise CBORDecodeError(
                         b'unexpected special value when '
                         b'expecting bytestring chunk: %d' % special
                     )
 
             # This value represents the non-initial chunk in an indefinite
             # length bytestring.
             elif self._state == self._STATE_WANT_BYTESTRING_CHUNK_SUBSEQUENT:
                 # We received a full chunk.
                 if special == SPECIAL_NONE:
                     self._decodedvalues.append(bytestringchunk(value))
 
                 # The end of stream marker.
                 elif special == SPECIAL_INDEFINITE_BREAK:
                     self._decodedvalues.append(bytestringchunk(b'', last=True))
 
                     # Since indefinite length bytestrings can't be used in
                     # collections, we must be at the root level.
                     assert not self._collectionstack
                     self._state = self._STATE_NONE
 
                 else:
                     raise CBORDecodeError(
                         b'unexpected special value when '
                         b'expecting bytestring chunk: %d' % special
                     )
 
             else:
                 raise CBORDecodeError(
                     b'unhandled decoder state: %d' % self._state
                 )
 
             # We could have just added the final value in a collection. End
             # all complete collections at the top of the stack.
             while True:
                 # Bail if we're not waiting on a new collection item.
                 if self._state not in (
                     self._STATE_WANT_ARRAY_VALUE,
                     self._STATE_WANT_MAP_KEY,
                     self._STATE_WANT_SET_VALUE,
                 ):
                     break
 
                 # Or we are expecting more items for this collection.
                 lastc = self._collectionstack[-1]
 
                 if lastc[b'remaining']:
                     break
 
                 # The collection at the top of the stack is complete.
 
                 # Discard it, as it isn't needed for future items.
                 self._collectionstack.pop()
 
                 # If this is a nested collection, we don't emit it, since it
                 # will be emitted by its parent collection. But we do need to
                 # update state to reflect what the new top-most collection
                 # on the stack is.
                 if self._collectionstack:
                     self._state = {
                         list: self._STATE_WANT_ARRAY_VALUE,
                         dict: self._STATE_WANT_MAP_KEY,
                         set: self._STATE_WANT_SET_VALUE,
                     }[type(self._collectionstack[-1][b'v'])]
 
                 # If this is the root collection, emit it.
                 else:
                     self._decodedvalues.append(lastc[b'v'])
                     self._state = self._STATE_NONE
 
         return (
             bool(self._decodedvalues),
             offset - initialoffset,
             0,
         )
 
     def getavailable(self):
         """Returns an iterator over fully decoded values.
 
         Once values are retrieved, they won't be available on the next call.
         """
 
         l = list(self._decodedvalues)
         self._decodedvalues = []
         return l
 
 
 class bufferingdecoder(object):
     """A CBOR decoder that buffers undecoded input.
 
     This is a glorified wrapper around ``sansiodecoder`` that adds a buffering
     layer. All input that isn't consumed by ``sansiodecoder`` will be buffered
     and concatenated with any new input that arrives later.
 
     TODO consider adding limits as to the maximum amount of data that can
     be buffered.
     """
 
     def __init__(self):
         self._decoder = sansiodecoder()
         self._chunks = []
         self._wanted = 0
 
     def decode(self, b):
         """Attempt to decode bytes to CBOR values.
 
         Returns a tuple with the following fields:
 
         * Bool indicating whether new values are available for retrieval.
         * Integer number of bytes decoded from the new input.
         * Integer number of bytes wanted to decode the next value.
         """
         # We /might/ be able to support passing a bytearray all the
         # way through. For now, let's cheat.
         if isinstance(b, bytearray):
             b = bytes(b)
 
         # Our strategy for buffering is to aggregate the incoming chunks in a
         # list until we've received enough data to decode the next item.
         # This is slightly more complicated than using an ``io.BytesIO``
         # or continuously concatenating incoming data. However, because it
         # isn't constantly reallocating backing memory for a growing buffer,
         # it prevents excessive memory thrashing and is significantly faster,
         # especially in cases where the percentage of input chunks that don't
         # decode into a full item is high.
 
         if self._chunks:
             # A previous call said we needed N bytes to decode the next item.
             # But this call doesn't provide enough data. We buffer the incoming
             # chunk without attempting to decode.
             if len(b) < self._wanted:
                 self._chunks.append(b)
                 self._wanted -= len(b)
                 return False, 0, self._wanted
 
             # Else we may have enough data to decode the next item. Aggregate
             # old data with new and reset the buffer.
             newlen = len(b)
             self._chunks.append(b)
             b = b''.join(self._chunks)
             self._chunks = []
             oldlen = len(b) - newlen
 
         else:
             oldlen = 0
 
         available, readcount, wanted = self._decoder.decode(b)
         self._wanted = wanted
 
         if readcount < len(b):
             self._chunks.append(b[readcount:])
 
         return available, readcount - oldlen, wanted
 
     def getavailable(self):
         return self._decoder.getavailable()
 
 
 def decodeall(b):
     """Decode all CBOR items present in an iterable of bytes.
 
     In addition to regular decode errors, raises CBORDecodeError if the
     entirety of the passed buffer does not fully decode to complete CBOR
     values. This includes failure to decode any value, incomplete collection
     types, incomplete indefinite length items, and extra data at the end of
     the buffer.
     """
     if not b:
         return []
 
     decoder = sansiodecoder()
 
     havevalues, readcount, wantbytes = decoder.decode(b)
 
     if readcount != len(b):
         raise CBORDecodeError(b'input data not fully consumed')
 
     if decoder.inprogress:
         raise CBORDecodeError(b'input data not complete')
 
     return decoder.getavailable()