revlog: signal which revlog index are compatible with Rust

Otherwise, Rust may treat python object like `cindex` object, leading to
trouble. The new attribute is an integer because I expect we might need a flag
field in the future.

As a start we get the rust code to raise a clear TypeError. We will use the
information in a smarter way in the next changesets.

Differential Revision: https://phab.mercurial-scm.org/D10665

mercurial/cext/parsers.c

@@ -1,763 +1,763
 /*
  parsers.c - efficient content parsing
 
  Copyright 2008 Olivia Mackall <[e-mail]> and others
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 */
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <ctype.h>
 #include <stddef.h>
 #include <string.h>
 
 #include "bitmanipulation.h"
 #include "charencode.h"
 #include "util.h"
 
 #ifdef IS_PY3K
 /* The mapping of Python types is meant to be temporary to get Python
  * 3 to compile. We should remove this once Python 3 support is fully
  * supported and proper types are used in the extensions themselves. */
 #define PyInt_Check PyLong_Check
 #define PyInt_FromLong PyLong_FromLong
 #define PyInt_FromSsize_t PyLong_FromSsize_t
 #define PyInt_AsLong PyLong_AsLong
 #endif
 
 static const char *const versionerrortext = "Python minor version mismatch";
 
 static PyObject *dict_new_presized(PyObject *self, PyObject *args)
 {
 	Py_ssize_t expected_size;
 
 	if (!PyArg_ParseTuple(args, "n:make_presized_dict", &expected_size)) {
 		return NULL;
 	}
 
 	return _dict_new_presized(expected_size);
 }
 
 static inline dirstateTupleObject *make_dirstate_tuple(char state, int mode,
                                                        int size, int mtime)
 {
 	dirstateTupleObject *t =
 	    PyObject_New(dirstateTupleObject, &dirstateTupleType);
 	if (!t) {
 		return NULL;
 	}
 	t->state = state;
 	t->mode = mode;
 	t->size = size;
 	t->mtime = mtime;
 	return t;
 }
 
 static PyObject *dirstate_tuple_new(PyTypeObject *subtype, PyObject *args,
                                     PyObject *kwds)
 {
 	/* We do all the initialization here and not a tp_init function because
 	 * dirstate_tuple is immutable. */
 	dirstateTupleObject *t;
 	char state;
 	int size, mode, mtime;
 	if (!PyArg_ParseTuple(args, "ciii", &state, &mode, &size, &mtime)) {
 		return NULL;
 	}
 
 	t = (dirstateTupleObject *)subtype->tp_alloc(subtype, 1);
 	if (!t) {
 		return NULL;
 	}
 	t->state = state;
 	t->mode = mode;
 	t->size = size;
 	t->mtime = mtime;
 
 	return (PyObject *)t;
 }
 
 static void dirstate_tuple_dealloc(PyObject *o)
 {
 	PyObject_Del(o);
 }
 
 static Py_ssize_t dirstate_tuple_length(PyObject *o)
 {
 	return 4;
 }
 
 static PyObject *dirstate_tuple_item(PyObject *o, Py_ssize_t i)
 {
 	dirstateTupleObject *t = (dirstateTupleObject *)o;
 	switch (i) {
 	case 0:
 		return PyBytes_FromStringAndSize(&t->state, 1);
 	case 1:
 		return PyInt_FromLong(t->mode);
 	case 2:
 		return PyInt_FromLong(t->size);
 	case 3:
 		return PyInt_FromLong(t->mtime);
 	default:
 		PyErr_SetString(PyExc_IndexError, "index out of range");
 		return NULL;
 	}
 }
 
 static PySequenceMethods dirstate_tuple_sq = {
     dirstate_tuple_length, /* sq_length */
     0,                     /* sq_concat */
     0,                     /* sq_repeat */
     dirstate_tuple_item,   /* sq_item */
     0,                     /* sq_ass_item */
     0,                     /* sq_contains */
     0,                     /* sq_inplace_concat */
     0                      /* sq_inplace_repeat */
 };
 
 PyTypeObject dirstateTupleType = {
     PyVarObject_HEAD_INIT(NULL, 0)      /* header */
     "dirstate_tuple",                   /* tp_name */
     sizeof(dirstateTupleObject),        /* tp_basicsize */
     0,                                  /* tp_itemsize */
     (destructor)dirstate_tuple_dealloc, /* tp_dealloc */
     0,                                  /* tp_print */
     0,                                  /* tp_getattr */
     0,                                  /* tp_setattr */
     0,                                  /* tp_compare */
     0,                                  /* tp_repr */
     0,                                  /* tp_as_number */
     &dirstate_tuple_sq,                 /* tp_as_sequence */
     0,                                  /* tp_as_mapping */
     0,                                  /* tp_hash  */
     0,                                  /* tp_call */
     0,                                  /* tp_str */
     0,                                  /* tp_getattro */
     0,                                  /* tp_setattro */
     0,                                  /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,                 /* tp_flags */
     "dirstate tuple",                   /* tp_doc */
     0,                                  /* tp_traverse */
     0,                                  /* tp_clear */
     0,                                  /* tp_richcompare */
     0,                                  /* tp_weaklistoffset */
     0,                                  /* tp_iter */
     0,                                  /* tp_iternext */
     0,                                  /* tp_methods */
     0,                                  /* tp_members */
     0,                                  /* tp_getset */
     0,                                  /* tp_base */
     0,                                  /* tp_dict */
     0,                                  /* tp_descr_get */
     0,                                  /* tp_descr_set */
     0,                                  /* tp_dictoffset */
     0,                                  /* tp_init */
     0,                                  /* tp_alloc */
     dirstate_tuple_new,                 /* tp_new */
 };
 
 static PyObject *parse_dirstate(PyObject *self, PyObject *args)
 {
 	PyObject *dmap, *cmap, *parents = NULL, *ret = NULL;
 	PyObject *fname = NULL, *cname = NULL, *entry = NULL;
 	char state, *cur, *str, *cpos;
 	int mode, size, mtime;
 	unsigned int flen, pos = 40;
 	Py_ssize_t len = 40;
 	Py_ssize_t readlen;
 
 	if (!PyArg_ParseTuple(
 	        args, PY23("O!O!s#:parse_dirstate", "O!O!y#:parse_dirstate"),
 	        &PyDict_Type, &dmap, &PyDict_Type, &cmap, &str, &readlen)) {
 		goto quit;
 	}
 
 	len = readlen;
 
 	/* read parents */
 	if (len < 40) {
 		PyErr_SetString(PyExc_ValueError,
 		                "too little data for parents");
 		goto quit;
 	}
 
 	parents = Py_BuildValue(PY23("s#s#", "y#y#"), str, (Py_ssize_t)20,
 	                        str + 20, (Py_ssize_t)20);
 	if (!parents) {
 		goto quit;
 	}
 
 	/* read filenames */
 	while (pos >= 40 && pos < len) {
 		if (pos + 17 > len) {
 			PyErr_SetString(PyExc_ValueError,
 			                "overflow in dirstate");
 			goto quit;
 		}
 		cur = str + pos;
 		/* unpack header */
 		state = *cur;
 		mode = getbe32(cur + 1);
 		size = getbe32(cur + 5);
 		mtime = getbe32(cur + 9);
 		flen = getbe32(cur + 13);
 		pos += 17;
 		cur += 17;
 		if (flen > len - pos) {
 			PyErr_SetString(PyExc_ValueError,
 			                "overflow in dirstate");
 			goto quit;
 		}
 
 		entry =
 		    (PyObject *)make_dirstate_tuple(state, mode, size, mtime);
 		cpos = memchr(cur, 0, flen);
 		if (cpos) {
 			fname = PyBytes_FromStringAndSize(cur, cpos - cur);
 			cname = PyBytes_FromStringAndSize(
 			    cpos + 1, flen - (cpos - cur) - 1);
 			if (!fname || !cname ||
 			    PyDict_SetItem(cmap, fname, cname) == -1 ||
 			    PyDict_SetItem(dmap, fname, entry) == -1) {
 				goto quit;
 			}
 			Py_DECREF(cname);
 		} else {
 			fname = PyBytes_FromStringAndSize(cur, flen);
 			if (!fname ||
 			    PyDict_SetItem(dmap, fname, entry) == -1) {
 				goto quit;
 			}
 		}
 		Py_DECREF(fname);
 		Py_DECREF(entry);
 		fname = cname = entry = NULL;
 		pos += flen;
 	}
 
 	ret = parents;
 	Py_INCREF(ret);
 quit:
 	Py_XDECREF(fname);
 	Py_XDECREF(cname);
 	Py_XDECREF(entry);
 	Py_XDECREF(parents);
 	return ret;
 }
 
 /*
  * Build a set of non-normal and other parent entries from the dirstate dmap
  */
 static PyObject *nonnormalotherparententries(PyObject *self, PyObject *args)
 {
 	PyObject *dmap, *fname, *v;
 	PyObject *nonnset = NULL, *otherpset = NULL, *result = NULL;
 	Py_ssize_t pos;
 
 	if (!PyArg_ParseTuple(args, "O!:nonnormalentries", &PyDict_Type,
 	                      &dmap)) {
 		goto bail;
 	}
 
 	nonnset = PySet_New(NULL);
 	if (nonnset == NULL) {
 		goto bail;
 	}
 
 	otherpset = PySet_New(NULL);
 	if (otherpset == NULL) {
 		goto bail;
 	}
 
 	pos = 0;
 	while (PyDict_Next(dmap, &pos, &fname, &v)) {
 		dirstateTupleObject *t;
 		if (!dirstate_tuple_check(v)) {
 			PyErr_SetString(PyExc_TypeError,
 			                "expected a dirstate tuple");
 			goto bail;
 		}
 		t = (dirstateTupleObject *)v;
 
 		if (t->state == 'n' && t->size == -2) {
 			if (PySet_Add(otherpset, fname) == -1) {
 				goto bail;
 			}
 		}
 
 		if (t->state == 'n' && t->mtime != -1) {
 			continue;
 		}
 		if (PySet_Add(nonnset, fname) == -1) {
 			goto bail;
 		}
 	}
 
 	result = Py_BuildValue("(OO)", nonnset, otherpset);
 	if (result == NULL) {
 		goto bail;
 	}
 	Py_DECREF(nonnset);
 	Py_DECREF(otherpset);
 	return result;
 bail:
 	Py_XDECREF(nonnset);
 	Py_XDECREF(otherpset);
 	Py_XDECREF(result);
 	return NULL;
 }
 
 /*
  * Efficiently pack a dirstate object into its on-disk format.
  */
 static PyObject *pack_dirstate(PyObject *self, PyObject *args)
 {
 	PyObject *packobj = NULL;
 	PyObject *map, *copymap, *pl, *mtime_unset = NULL;
 	Py_ssize_t nbytes, pos, l;
 	PyObject *k, *v = NULL, *pn;
 	char *p, *s;
 	int now;
 
 	if (!PyArg_ParseTuple(args, "O!O!O!i:pack_dirstate", &PyDict_Type, &map,
 	                      &PyDict_Type, &copymap, &PyTuple_Type, &pl,
 	                      &now)) {
 		return NULL;
 	}
 
 	if (PyTuple_Size(pl) != 2) {
 		PyErr_SetString(PyExc_TypeError, "expected 2-element tuple");
 		return NULL;
 	}
 
 	/* Figure out how much we need to allocate. */
 	for (nbytes = 40, pos = 0; PyDict_Next(map, &pos, &k, &v);) {
 		PyObject *c;
 		if (!PyBytes_Check(k)) {
 			PyErr_SetString(PyExc_TypeError, "expected string key");
 			goto bail;
 		}
 		nbytes += PyBytes_GET_SIZE(k) + 17;
 		c = PyDict_GetItem(copymap, k);
 		if (c) {
 			if (!PyBytes_Check(c)) {
 				PyErr_SetString(PyExc_TypeError,
 				                "expected string key");
 				goto bail;
 			}
 			nbytes += PyBytes_GET_SIZE(c) + 1;
 		}
 	}
 
 	packobj = PyBytes_FromStringAndSize(NULL, nbytes);
 	if (packobj == NULL) {
 		goto bail;
 	}
 
 	p = PyBytes_AS_STRING(packobj);
 
 	pn = PyTuple_GET_ITEM(pl, 0);
 	if (PyBytes_AsStringAndSize(pn, &s, &l) == -1 || l != 20) {
 		PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash");
 		goto bail;
 	}
 	memcpy(p, s, l);
 	p += 20;
 	pn = PyTuple_GET_ITEM(pl, 1);
 	if (PyBytes_AsStringAndSize(pn, &s, &l) == -1 || l != 20) {
 		PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash");
 		goto bail;
 	}
 	memcpy(p, s, l);
 	p += 20;
 
 	for (pos = 0; PyDict_Next(map, &pos, &k, &v);) {
 		dirstateTupleObject *tuple;
 		char state;
 		int mode, size, mtime;
 		Py_ssize_t len, l;
 		PyObject *o;
 		char *t;
 
 		if (!dirstate_tuple_check(v)) {
 			PyErr_SetString(PyExc_TypeError,
 			                "expected a dirstate tuple");
 			goto bail;
 		}
 		tuple = (dirstateTupleObject *)v;
 
 		state = tuple->state;
 		mode = tuple->mode;
 		size = tuple->size;
 		mtime = tuple->mtime;
 		if (state == 'n' && mtime == now) {
 			/* See pure/parsers.py:pack_dirstate for why we do
 			 * this. */
 			mtime = -1;
 			mtime_unset = (PyObject *)make_dirstate_tuple(
 			    state, mode, size, mtime);
 			if (!mtime_unset) {
 				goto bail;
 			}
 			if (PyDict_SetItem(map, k, mtime_unset) == -1) {
 				goto bail;
 			}
 			Py_DECREF(mtime_unset);
 			mtime_unset = NULL;
 		}
 		*p++ = state;
 		putbe32((uint32_t)mode, p);
 		putbe32((uint32_t)size, p + 4);
 		putbe32((uint32_t)mtime, p + 8);
 		t = p + 12;
 		p += 16;
 		len = PyBytes_GET_SIZE(k);
 		memcpy(p, PyBytes_AS_STRING(k), len);
 		p += len;
 		o = PyDict_GetItem(copymap, k);
 		if (o) {
 			*p++ = '\0';
 			l = PyBytes_GET_SIZE(o);
 			memcpy(p, PyBytes_AS_STRING(o), l);
 			p += l;
 			len += l + 1;
 		}
 		putbe32((uint32_t)len, t);
 	}
 
 	pos = p - PyBytes_AS_STRING(packobj);
 	if (pos != nbytes) {
 		PyErr_Format(PyExc_SystemError, "bad dirstate size: %ld != %ld",
 		             (long)pos, (long)nbytes);
 		goto bail;
 	}
 
 	return packobj;
 bail:
 	Py_XDECREF(mtime_unset);
 	Py_XDECREF(packobj);
 	Py_XDECREF(v);
 	return NULL;
 }
 
 #define BUMPED_FIX 1
 #define USING_SHA_256 2
 #define FM1_HEADER_SIZE (4 + 8 + 2 + 2 + 1 + 1 + 1)
 
 static PyObject *readshas(const char *source, unsigned char num,
                           Py_ssize_t hashwidth)
 {
 	int i;
 	PyObject *list = PyTuple_New(num);
 	if (list == NULL) {
 		return NULL;
 	}
 	for (i = 0; i < num; i++) {
 		PyObject *hash = PyBytes_FromStringAndSize(source, hashwidth);
 		if (hash == NULL) {
 			Py_DECREF(list);
 			return NULL;
 		}
 		PyTuple_SET_ITEM(list, i, hash);
 		source += hashwidth;
 	}
 	return list;
 }
 
 static PyObject *fm1readmarker(const char *databegin, const char *dataend,
                                uint32_t *msize)
 {
 	const char *data = databegin;
 	const char *meta;
 
 	double mtime;
 	int16_t tz;
 	uint16_t flags;
 	unsigned char nsuccs, nparents, nmetadata;
 	Py_ssize_t hashwidth = 20;
 
 	PyObject *prec = NULL, *parents = NULL, *succs = NULL;
 	PyObject *metadata = NULL, *ret = NULL;
 	int i;
 
 	if (data + FM1_HEADER_SIZE > dataend) {
 		goto overflow;
 	}
 
 	*msize = getbe32(data);
 	data += 4;
 	mtime = getbefloat64(data);
 	data += 8;
 	tz = getbeint16(data);
 	data += 2;
 	flags = getbeuint16(data);
 	data += 2;
 
 	if (flags & USING_SHA_256) {
 		hashwidth = 32;
 	}
 
 	nsuccs = (unsigned char)(*data++);
 	nparents = (unsigned char)(*data++);
 	nmetadata = (unsigned char)(*data++);
 
 	if (databegin + *msize > dataend) {
 		goto overflow;
 	}
 	dataend = databegin + *msize; /* narrow down to marker size */
 
 	if (data + hashwidth > dataend) {
 		goto overflow;
 	}
 	prec = PyBytes_FromStringAndSize(data, hashwidth);
 	data += hashwidth;
 	if (prec == NULL) {
 		goto bail;
 	}
 
 	if (data + nsuccs * hashwidth > dataend) {
 		goto overflow;
 	}
 	succs = readshas(data, nsuccs, hashwidth);
 	if (succs == NULL) {
 		goto bail;
 	}
 	data += nsuccs * hashwidth;
 
 	if (nparents == 1 || nparents == 2) {
 		if (data + nparents * hashwidth > dataend) {
 			goto overflow;
 		}
 		parents = readshas(data, nparents, hashwidth);
 		if (parents == NULL) {
 			goto bail;
 		}
 		data += nparents * hashwidth;
 	} else {
 		parents = Py_None;
 		Py_INCREF(parents);
 	}
 
 	if (data + 2 * nmetadata > dataend) {
 		goto overflow;
 	}
 	meta = data + (2 * nmetadata);
 	metadata = PyTuple_New(nmetadata);
 	if (metadata == NULL) {
 		goto bail;
 	}
 	for (i = 0; i < nmetadata; i++) {
 		PyObject *tmp, *left = NULL, *right = NULL;
 		Py_ssize_t leftsize = (unsigned char)(*data++);
 		Py_ssize_t rightsize = (unsigned char)(*data++);
 		if (meta + leftsize + rightsize > dataend) {
 			goto overflow;
 		}
 		left = PyBytes_FromStringAndSize(meta, leftsize);
 		meta += leftsize;
 		right = PyBytes_FromStringAndSize(meta, rightsize);
 		meta += rightsize;
 		tmp = PyTuple_New(2);
 		if (!left || !right || !tmp) {
 			Py_XDECREF(left);
 			Py_XDECREF(right);
 			Py_XDECREF(tmp);
 			goto bail;
 		}
 		PyTuple_SET_ITEM(tmp, 0, left);
 		PyTuple_SET_ITEM(tmp, 1, right);
 		PyTuple_SET_ITEM(metadata, i, tmp);
 	}
 	ret = Py_BuildValue("(OOHO(di)O)", prec, succs, flags, metadata, mtime,
 	                    (int)tz * 60, parents);
 	goto bail; /* return successfully */
 
 overflow:
 	PyErr_SetString(PyExc_ValueError, "overflow in obsstore");
 bail:
 	Py_XDECREF(prec);
 	Py_XDECREF(succs);
 	Py_XDECREF(metadata);
 	Py_XDECREF(parents);
 	return ret;
 }
 
 static PyObject *fm1readmarkers(PyObject *self, PyObject *args)
 {
 	const char *data, *dataend;
 	Py_ssize_t datalen, offset, stop;
 	PyObject *markers = NULL;
 
 	if (!PyArg_ParseTuple(args, PY23("s#nn", "y#nn"), &data, &datalen,
 	                      &offset, &stop)) {
 		return NULL;
 	}
 	if (offset < 0) {
 		PyErr_SetString(PyExc_ValueError,
 		                "invalid negative offset in fm1readmarkers");
 		return NULL;
 	}
 	if (stop > datalen) {
 		PyErr_SetString(
 		    PyExc_ValueError,
 		    "stop longer than data length in fm1readmarkers");
 		return NULL;
 	}
 	dataend = data + datalen;
 	data += offset;
 	markers = PyList_New(0);
 	if (!markers) {
 		return NULL;
 	}
 	while (offset < stop) {
 		uint32_t msize;
 		int error;
 		PyObject *record = fm1readmarker(data, dataend, &msize);
 		if (!record) {
 			goto bail;
 		}
 		error = PyList_Append(markers, record);
 		Py_DECREF(record);
 		if (error) {
 			goto bail;
 		}
 		data += msize;
 		offset += msize;
 	}
 	return markers;
 bail:
 	Py_DECREF(markers);
 	return NULL;
 }
 
 static char parsers_doc[] = "Efficient content parsing.";
 
 PyObject *encodedir(PyObject *self, PyObject *args);
 PyObject *pathencode(PyObject *self, PyObject *args);
 PyObject *lowerencode(PyObject *self, PyObject *args);
 PyObject *parse_index2(PyObject *self, PyObject *args, PyObject *kwargs);
 
 static PyMethodDef methods[] = {
     {"pack_dirstate", pack_dirstate, METH_VARARGS, "pack a dirstate\n"},
     {"nonnormalotherparententries", nonnormalotherparententries, METH_VARARGS,
      "create a set containing non-normal and other parent entries of given "
      "dirstate\n"},
     {"parse_dirstate", parse_dirstate, METH_VARARGS, "parse a dirstate\n"},
     {"parse_index2", (PyCFunction)parse_index2, METH_VARARGS | METH_KEYWORDS,
      "parse a revlog index\n"},
     {"isasciistr", isasciistr, METH_VARARGS, "check if an ASCII string\n"},
     {"asciilower", asciilower, METH_VARARGS, "lowercase an ASCII string\n"},
     {"asciiupper", asciiupper, METH_VARARGS, "uppercase an ASCII string\n"},
     {"dict_new_presized", dict_new_presized, METH_VARARGS,
      "construct a dict with an expected size\n"},
     {"make_file_foldmap", make_file_foldmap, METH_VARARGS,
      "make file foldmap\n"},
     {"jsonescapeu8fast", jsonescapeu8fast, METH_VARARGS,
      "escape a UTF-8 byte string to JSON (fast path)\n"},
     {"encodedir", encodedir, METH_VARARGS, "encodedir a path\n"},
     {"pathencode", pathencode, METH_VARARGS, "fncache-encode a path\n"},
     {"lowerencode", lowerencode, METH_VARARGS, "lower-encode a path\n"},
     {"fm1readmarkers", fm1readmarkers, METH_VARARGS,
      "parse v1 obsolete markers\n"},
     {NULL, NULL}};
 
 void dirs_module_init(PyObject *mod);
 void manifest_module_init(PyObject *mod);
 void revlog_module_init(PyObject *mod);
 
-static const int version = 19;
+static const int version = 20;
 
 static void module_init(PyObject *mod)
 {
 	PyObject *capsule = NULL;
 	PyModule_AddIntConstant(mod, "version", version);
 
 	/* This module constant has two purposes.  First, it lets us unit test
 	 * the ImportError raised without hard-coding any error text.  This
 	 * means we can change the text in the future without breaking tests,
 	 * even across changesets without a recompile.  Second, its presence
 	 * can be used to determine whether the version-checking logic is
 	 * present, which also helps in testing across changesets without a
 	 * recompile.  Note that this means the pure-Python version of parsers
 	 * should not have this module constant. */
 	PyModule_AddStringConstant(mod, "versionerrortext", versionerrortext);
 
 	dirs_module_init(mod);
 	manifest_module_init(mod);
 	revlog_module_init(mod);
 
 	capsule = PyCapsule_New(
 	    make_dirstate_tuple,
 	    "mercurial.cext.parsers.make_dirstate_tuple_CAPI", NULL);
 	if (capsule != NULL)
 		PyModule_AddObject(mod, "make_dirstate_tuple_CAPI", capsule);
 
 	if (PyType_Ready(&dirstateTupleType) < 0) {
 		return;
 	}
 	Py_INCREF(&dirstateTupleType);
 	PyModule_AddObject(mod, "dirstatetuple",
 	                   (PyObject *)&dirstateTupleType);
 }
 
 static int check_python_version(void)
 {
 	PyObject *sys = PyImport_ImportModule("sys"), *ver;
 	long hexversion;
 	if (!sys) {
 		return -1;
 	}
 	ver = PyObject_GetAttrString(sys, "hexversion");
 	Py_DECREF(sys);
 	if (!ver) {
 		return -1;
 	}
 	hexversion = PyInt_AsLong(ver);
 	Py_DECREF(ver);
 	/* sys.hexversion is a 32-bit number by default, so the -1 case
 	 * should only occur in unusual circumstances (e.g. if sys.hexversion
 	 * is manually set to an invalid value). */
 	if ((hexversion == -1) || (hexversion >> 16 != PY_VERSION_HEX >> 16)) {
 		PyErr_Format(PyExc_ImportError,
 		             "%s: The Mercurial extension "
 		             "modules were compiled with Python " PY_VERSION
 		             ", but "
 		             "Mercurial is currently using Python with "
 		             "sys.hexversion=%ld: "
 		             "Python %s\n at: %s",
 		             versionerrortext, hexversion, Py_GetVersion(),
 		             Py_GetProgramFullPath());
 		return -1;
 	}
 	return 0;
 }
 
 #ifdef IS_PY3K
 static struct PyModuleDef parsers_module = {PyModuleDef_HEAD_INIT, "parsers",
                                             parsers_doc, -1, methods};
 
 PyMODINIT_FUNC PyInit_parsers(void)
 {
 	PyObject *mod;
 
 	if (check_python_version() == -1)
 		return NULL;
 	mod = PyModule_Create(&parsers_module);
 	module_init(mod);
 	return mod;
 }
 #else
 PyMODINIT_FUNC initparsers(void)
 {
 	PyObject *mod;
 
 	if (check_python_version() == -1) {
 		return;
 	}
 	mod = Py_InitModule3("parsers", methods, parsers_doc);
 	module_init(mod);
 }
 #endif

mercurial/cext/revlog.c

@@ -1,3055 +1,3060
 /*
  parsers.c - efficient content parsing
 
  Copyright 2008 Olivia Mackall <[e-mail]> and others
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 */
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <assert.h>
 #include <ctype.h>
 #include <limits.h>
 #include <stddef.h>
 #include <stdlib.h>
 #include <string.h>
 #include <structmember.h>
 
 #include "bitmanipulation.h"
 #include "charencode.h"
 #include "compat.h"
 #include "revlog.h"
 #include "util.h"
 
 #ifdef IS_PY3K
 /* The mapping of Python types is meant to be temporary to get Python
  * 3 to compile. We should remove this once Python 3 support is fully
  * supported and proper types are used in the extensions themselves. */
 #define PyInt_Check PyLong_Check
 #define PyInt_FromLong PyLong_FromLong
 #define PyInt_FromSsize_t PyLong_FromSsize_t
 #define PyInt_AsLong PyLong_AsLong
 #endif
 
 typedef struct indexObjectStruct indexObject;
 
 typedef struct {
 	int children[16];
 } nodetreenode;
 
 typedef struct {
 	int abi_version;
 	Py_ssize_t (*index_length)(const indexObject *);
 	const char *(*index_node)(indexObject *, Py_ssize_t);
 	int (*index_parents)(PyObject *, int, int *);
 } Revlog_CAPI;
 
 /*
  * A base-16 trie for fast node->rev mapping.
  *
  * Positive value is index of the next node in the trie
  * Negative value is a leaf: -(rev + 2)
  * Zero is empty
  */
 typedef struct {
 	indexObject *index;
 	nodetreenode *nodes;
 	Py_ssize_t nodelen;
 	size_t length;   /* # nodes in use */
 	size_t capacity; /* # nodes allocated */
 	int depth;       /* maximum depth of tree */
 	int splits;      /* # splits performed */
 } nodetree;
 
 typedef struct {
 	PyObject_HEAD /* ; */
 	    nodetree nt;
 } nodetreeObject;
 
 /*
  * This class has two behaviors.
  *
  * When used in a list-like way (with integer keys), we decode an
  * entry in a RevlogNG index file on demand. We have limited support for
  * integer-keyed insert and delete, only at elements right before the
  * end.
  *
  * With string keys, we lazily perform a reverse mapping from node to
  * rev, using a base-16 trie.
  */
 struct indexObjectStruct {
 	PyObject_HEAD
 	    /* Type-specific fields go here. */
 	    PyObject *data;     /* raw bytes of index */
 	Py_ssize_t nodelen;     /* digest size of the hash, 20 for SHA-1 */
 	PyObject *nullentry;    /* fast path for references to null */
 	Py_buffer buf;          /* buffer of data */
 	const char **offsets;   /* populated on demand */
 	Py_ssize_t length;      /* current on-disk number of elements */
 	unsigned new_length;    /* number of added elements */
 	unsigned added_length;  /* space reserved for added elements */
 	char *added;            /* populated on demand */
 	PyObject *headrevs;     /* cache, invalidated on changes */
 	PyObject *filteredrevs; /* filtered revs set */
 	nodetree nt;            /* base-16 trie */
 	int ntinitialized;      /* 0 or 1 */
 	int ntrev;              /* last rev scanned */
 	int ntlookups;          /* # lookups */
 	int ntmisses;           /* # lookups that miss the cache */
 	int inlined;
 	long entry_size; /* size of index headers. Differs in v1 v.s. v2 format
 	                  */
-	char format_version; /* size of index headers. Differs in v1 v.s. v2
-	                        format */
+	long rust_ext_compat; /* compatibility with being used in rust
+	                         extensions */
+	char format_version;  /* size of index headers. Differs in v1 v.s. v2
+	                         format */
 };
 
 static Py_ssize_t index_length(const indexObject *self)
 {
 	return self->length + self->new_length;
 }
 
 static const char nullid[32] = {0};
 static const Py_ssize_t nullrev = -1;
 
 static Py_ssize_t inline_scan(indexObject *self, const char **offsets);
 
 static int index_find_node(indexObject *self, const char *node);
 
 #if LONG_MAX == 0x7fffffffL
 static const char *const tuple_format = PY23("Kiiiiiis#KiBB", "Kiiiiiiy#KiBB");
 #else
 static const char *const tuple_format = PY23("kiiiiiis#kiBB", "kiiiiiiy#kiBB");
 #endif
 
 /* A RevlogNG v1 index entry is 64 bytes long. */
 static const long v1_entry_size = 64;
 
 /* A Revlogv2 index entry is 96 bytes long. */
 static const long v2_entry_size = 96;
 
 static const long format_v1 = 1; /* Internal only, could be any number */
 static const long format_v2 = 2; /* Internal only, could be any number */
 
 static const char comp_mode_inline = 2;
 
 static void raise_revlog_error(void)
 {
 	PyObject *mod = NULL, *dict = NULL, *errclass = NULL;
 
 	mod = PyImport_ImportModule("mercurial.error");
 	if (mod == NULL) {
 		goto cleanup;
 	}
 
 	dict = PyModule_GetDict(mod);
 	if (dict == NULL) {
 		goto cleanup;
 	}
 	Py_INCREF(dict);
 
 	errclass = PyDict_GetItemString(dict, "RevlogError");
 	if (errclass == NULL) {
 		PyErr_SetString(PyExc_SystemError,
 		                "could not find RevlogError");
 		goto cleanup;
 	}
 
 	/* value of exception is ignored by callers */
 	PyErr_SetString(errclass, "RevlogError");
 
 cleanup:
 	Py_XDECREF(dict);
 	Py_XDECREF(mod);
 }
 
 /*
  * Return a pointer to the beginning of a RevlogNG record.
  */
 static const char *index_deref(indexObject *self, Py_ssize_t pos)
 {
 	if (pos >= self->length)
 		return self->added + (pos - self->length) * self->entry_size;
 
 	if (self->inlined && pos > 0) {
 		if (self->offsets == NULL) {
 			Py_ssize_t ret;
 			self->offsets =
 			    PyMem_Malloc(self->length * sizeof(*self->offsets));
 			if (self->offsets == NULL)
 				return (const char *)PyErr_NoMemory();
 			ret = inline_scan(self, self->offsets);
 			if (ret == -1) {
 				return NULL;
 			};
 		}
 		return self->offsets[pos];
 	}
 
 	return (const char *)(self->buf.buf) + pos * self->entry_size;
 }
 
 /*
  * Get parents of the given rev.
  *
  * The specified rev must be valid and must not be nullrev. A returned
  * parent revision may be nullrev, but is guaranteed to be in valid range.
  */
 static inline int index_get_parents(indexObject *self, Py_ssize_t rev, int *ps,
                                     int maxrev)
 {
 	const char *data = index_deref(self, rev);
 
 	ps[0] = getbe32(data + 24);
 	ps[1] = getbe32(data + 28);
 
 	/* If index file is corrupted, ps[] may point to invalid revisions. So
 	 * there is a risk of buffer overflow to trust them unconditionally. */
 	if (ps[0] < -1 || ps[0] > maxrev || ps[1] < -1 || ps[1] > maxrev) {
 		PyErr_SetString(PyExc_ValueError, "parent out of range");
 		return -1;
 	}
 	return 0;
 }
 
 /*
  * Get parents of the given rev.
  *
  * If the specified rev is out of range, IndexError will be raised. If the
  * revlog entry is corrupted, ValueError may be raised.
  *
  * Returns 0 on success or -1 on failure.
  */
 static int HgRevlogIndex_GetParents(PyObject *op, int rev, int *ps)
 {
 	int tiprev;
 	if (!op || !HgRevlogIndex_Check(op) || !ps) {
 		PyErr_BadInternalCall();
 		return -1;
 	}
 	tiprev = (int)index_length((indexObject *)op) - 1;
 	if (rev < -1 || rev > tiprev) {
 		PyErr_Format(PyExc_IndexError, "rev out of range: %d", rev);
 		return -1;
 	} else if (rev == -1) {
 		ps[0] = ps[1] = -1;
 		return 0;
 	} else {
 		return index_get_parents((indexObject *)op, rev, ps, tiprev);
 	}
 }
 
 static inline int64_t index_get_start(indexObject *self, Py_ssize_t rev)
 {
 	const char *data;
 	uint64_t offset;
 
 	if (rev == nullrev)
 		return 0;
 
 	data = index_deref(self, rev);
 	offset = getbe32(data + 4);
 	if (rev == 0) {
 		/* mask out version number for the first entry */
 		offset &= 0xFFFF;
 	} else {
 		uint32_t offset_high = getbe32(data);
 		offset |= ((uint64_t)offset_high) << 32;
 	}
 	return (int64_t)(offset >> 16);
 }
 
 static inline int index_get_length(indexObject *self, Py_ssize_t rev)
 {
 	const char *data;
 	int tmp;
 
 	if (rev == nullrev)
 		return 0;
 
 	data = index_deref(self, rev);
 
 	tmp = (int)getbe32(data + 8);
 	if (tmp < 0) {
 		PyErr_Format(PyExc_OverflowError,
 		             "revlog entry size out of bound (%d)", tmp);
 		return -1;
 	}
 	return tmp;
 }
 
 /*
  * RevlogNG format (all in big endian, data may be inlined):
  *    6 bytes: offset
  *    2 bytes: flags
  *    4 bytes: compressed length
  *    4 bytes: uncompressed length
  *    4 bytes: base revision
  *    4 bytes: link revision
  *    4 bytes: parent 1 revision
  *    4 bytes: parent 2 revision
  *   32 bytes: nodeid (only 20 bytes used with SHA-1)
  */
 static PyObject *index_get(indexObject *self, Py_ssize_t pos)
 {
 	uint64_t offset_flags, sidedata_offset;
 	int comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2,
 	    sidedata_comp_len;
 	char data_comp_mode, sidedata_comp_mode;
 	const char *c_node_id;
 	const char *data;
 	Py_ssize_t length = index_length(self);
 
 	if (pos == nullrev) {
 		Py_INCREF(self->nullentry);
 		return self->nullentry;
 	}
 
 	if (pos < 0 || pos >= length) {
 		PyErr_SetString(PyExc_IndexError, "revlog index out of range");
 		return NULL;
 	}
 
 	data = index_deref(self, pos);
 	if (data == NULL)
 		return NULL;
 
 	offset_flags = getbe32(data + 4);
 	/*
 	 * The first entry on-disk needs the version number masked out,
 	 * but this doesn't apply if entries are added to an empty index.
 	 */
 	if (self->length && pos == 0)
 		offset_flags &= 0xFFFF;
 	else {
 		uint32_t offset_high = getbe32(data);
 		offset_flags |= ((uint64_t)offset_high) << 32;
 	}
 
 	comp_len = getbe32(data + 8);
 	uncomp_len = getbe32(data + 12);
 	base_rev = getbe32(data + 16);
 	link_rev = getbe32(data + 20);
 	parent_1 = getbe32(data + 24);
 	parent_2 = getbe32(data + 28);
 	c_node_id = data + 32;
 
 	if (self->format_version == format_v1) {
 		sidedata_offset = 0;
 		sidedata_comp_len = 0;
 		data_comp_mode = comp_mode_inline;
 		sidedata_comp_mode = comp_mode_inline;
 	} else {
 		sidedata_offset = getbe64(data + 64);
 		sidedata_comp_len = getbe32(data + 72);
 		data_comp_mode = data[76] & 3;
 		sidedata_comp_mode = ((data[76] >> 2) & 3);
 	}
 
 	return Py_BuildValue(tuple_format, offset_flags, comp_len, uncomp_len,
 	                     base_rev, link_rev, parent_1, parent_2, c_node_id,
 	                     self->nodelen, sidedata_offset, sidedata_comp_len,
 	                     data_comp_mode, sidedata_comp_mode);
 }
 /*
  * Pack header information in binary
  */
 static PyObject *index_pack_header(indexObject *self, PyObject *args)
 {
 	int header;
 	char out[4];
 	if (!PyArg_ParseTuple(args, "I", &header)) {
 		return NULL;
 	}
 	if (self->format_version != format_v1) {
 		PyErr_Format(PyExc_RuntimeError,
 		             "version header should go in the docket, not the "
 		             "index: %lu",
 		             header);
 		return NULL;
 	}
 	putbe32(header, out);
 	return PyBytes_FromStringAndSize(out, 4);
 }
 /*
  * Return the raw binary string representing a revision
  */
 static PyObject *index_entry_binary(indexObject *self, PyObject *value)
 {
 	long rev;
 	const char *data;
 	Py_ssize_t length = index_length(self);
 
 	if (!pylong_to_long(value, &rev)) {
 		return NULL;
 	}
 	if (rev < 0 || rev >= length) {
 		PyErr_Format(PyExc_ValueError, "revlog index out of range: %ld",
 		             rev);
 		return NULL;
 	};
 
 	data = index_deref(self, rev);
 	if (data == NULL)
 		return NULL;
 	if (rev == 0 && self->format_version == format_v1) {
 		/* the header is eating the start of the first entry */
 		return PyBytes_FromStringAndSize(data + 4,
 		                                 self->entry_size - 4);
 	}
 	return PyBytes_FromStringAndSize(data, self->entry_size);
 }
 
 /*
  * Return the hash of node corresponding to the given rev.
  */
 static const char *index_node(indexObject *self, Py_ssize_t pos)
 {
 	Py_ssize_t length = index_length(self);
 	const char *data;
 
 	if (pos == nullrev)
 		return nullid;
 
 	if (pos >= length)
 		return NULL;
 
 	data = index_deref(self, pos);
 	return data ? data + 32 : NULL;
 }
 
 /*
  * Return the hash of the node corresponding to the given rev. The
  * rev is assumed to be existing. If not, an exception is set.
  */
 static const char *index_node_existing(indexObject *self, Py_ssize_t pos)
 {
 	const char *node = index_node(self, pos);
 	if (node == NULL) {
 		PyErr_Format(PyExc_IndexError, "could not access rev %d",
 		             (int)pos);
 	}
 	return node;
 }
 
 static int nt_insert(nodetree *self, const char *node, int rev);
 
 static int node_check(Py_ssize_t nodelen, PyObject *obj, char **node)
 {
 	Py_ssize_t thisnodelen;
 	if (PyBytes_AsStringAndSize(obj, node, &thisnodelen) == -1)
 		return -1;
 	if (nodelen == thisnodelen)
 		return 0;
 	PyErr_Format(PyExc_ValueError, "node len %zd != expected node len %zd",
 	             thisnodelen, nodelen);
 	return -1;
 }
 
 static PyObject *index_append(indexObject *self, PyObject *obj)
 {
 	uint64_t offset_flags, sidedata_offset;
 	int rev, comp_len, uncomp_len, base_rev, link_rev, parent_1, parent_2;
 	char data_comp_mode, sidedata_comp_mode;
 	Py_ssize_t c_node_id_len, sidedata_comp_len;
 	const char *c_node_id;
 	char comp_field;
 	char *data;
 
 	if (!PyArg_ParseTuple(obj, tuple_format, &offset_flags, &comp_len,
 	                      &uncomp_len, &base_rev, &link_rev, &parent_1,
 	                      &parent_2, &c_node_id, &c_node_id_len,
 	                      &sidedata_offset, &sidedata_comp_len,
 	                      &data_comp_mode, &sidedata_comp_mode)) {
 		PyErr_SetString(PyExc_TypeError, "11-tuple required");
 		return NULL;
 	}
 
 	if (c_node_id_len != self->nodelen) {
 		PyErr_SetString(PyExc_TypeError, "invalid node");
 		return NULL;
 	}
 	if (self->format_version == format_v1) {
 
 		if (data_comp_mode != comp_mode_inline) {
 			PyErr_Format(PyExc_ValueError,
 			             "invalid data compression mode: %i",
 			             data_comp_mode);
 			return NULL;
 		}
 		if (sidedata_comp_mode != comp_mode_inline) {
 			PyErr_Format(PyExc_ValueError,
 			             "invalid sidedata compression mode: %i",
 			             sidedata_comp_mode);
 			return NULL;
 		}
 	}
 
 	if (self->new_length == self->added_length) {
 		size_t new_added_length =
 		    self->added_length ? self->added_length * 2 : 4096;
 		void *new_added = PyMem_Realloc(
 		    self->added, new_added_length * self->entry_size);
 		if (!new_added)
 			return PyErr_NoMemory();
 		self->added = new_added;
 		self->added_length = new_added_length;
 	}
 	rev = self->length + self->new_length;
 	data = self->added + self->entry_size * self->new_length++;
 	putbe32(offset_flags >> 32, data);
 	putbe32(offset_flags & 0xffffffffU, data + 4);
 	putbe32(comp_len, data + 8);
 	putbe32(uncomp_len, data + 12);
 	putbe32(base_rev, data + 16);
 	putbe32(link_rev, data + 20);
 	putbe32(parent_1, data + 24);
 	putbe32(parent_2, data + 28);
 	memcpy(data + 32, c_node_id, c_node_id_len);
 	/* Padding since SHA-1 is only 20 bytes for now */
 	memset(data + 32 + c_node_id_len, 0, 32 - c_node_id_len);
 	if (self->format_version == format_v2) {
 		putbe64(sidedata_offset, data + 64);
 		putbe32(sidedata_comp_len, data + 72);
 		comp_field = data_comp_mode & 3;
 		comp_field = comp_field | (sidedata_comp_mode & 3) << 2;
 		data[76] = comp_field;
 		/* Padding for 96 bytes alignment */
 		memset(data + 77, 0, self->entry_size - 77);
 	}
 
 	if (self->ntinitialized)
 		nt_insert(&self->nt, c_node_id, rev);
 
 	Py_CLEAR(self->headrevs);
 	Py_RETURN_NONE;
 }
 
 /* Replace an existing index entry's sidedata offset and length with new ones.
    This cannot be used outside of the context of sidedata rewriting,
    inside the transaction that creates the given revision. */
 static PyObject *index_replace_sidedata_info(indexObject *self, PyObject *args)
 {
 	uint64_t offset_flags, sidedata_offset;
 	int rev;
 	char comp_mode;
 	Py_ssize_t sidedata_comp_len;
 	char *data;
 #if LONG_MAX == 0x7fffffffL
 	const char *const sidedata_format = PY23("nKiKB", "nKiKB");
 #else
 	const char *const sidedata_format = PY23("nkikB", "nkikB");
 #endif
 
 	if (self->entry_size == v1_entry_size || self->inlined) {
 		/*
 		 There is a bug in the transaction handling when going from an
 	   inline revlog to a separate index and data file. Turn it off until
 	   it's fixed, since v2 revlogs sometimes get rewritten on exchange.
 	   See issue6485.
 	  */
 		raise_revlog_error();
 		return NULL;
 	}
 
 	if (!PyArg_ParseTuple(args, sidedata_format, &rev, &sidedata_offset,
 	                      &sidedata_comp_len, &offset_flags, &comp_mode))
 		return NULL;
 
 	if (rev < 0 || rev >= index_length(self)) {
 		PyErr_SetString(PyExc_IndexError, "revision outside index");
 		return NULL;
 	}
 	if (rev < self->length) {
 		PyErr_SetString(
 		    PyExc_IndexError,
 		    "cannot rewrite entries outside of this transaction");
 		return NULL;
 	}
 
 	/* Find the newly added node, offset from the "already on-disk" length
 	 */
 	data = self->added + self->entry_size * (rev - self->length);
 	putbe64(offset_flags, data);
 	putbe64(sidedata_offset, data + 64);
 	putbe32(sidedata_comp_len, data + 72);
 	data[76] = (data[76] & ~(3 << 2)) | ((comp_mode & 3) << 2);
 
 	Py_RETURN_NONE;
 }
 
 static PyObject *index_stats(indexObject *self)
 {
 	PyObject *obj = PyDict_New();
 	PyObject *s = NULL;
 	PyObject *t = NULL;
 
 	if (obj == NULL)
 		return NULL;
 
 #define istat(__n, __d)                                                        \
 	do {                                                                   \
 		s = PyBytes_FromString(__d);                                   \
 		t = PyInt_FromSsize_t(self->__n);                              \
 		if (!s || !t)                                                  \
 			goto bail;                                             \
 		if (PyDict_SetItem(obj, s, t) == -1)                           \
 			goto bail;                                             \
 		Py_CLEAR(s);                                                   \
 		Py_CLEAR(t);                                                   \
 	} while (0)
 
 	if (self->added_length)
 		istat(new_length, "index entries added");
 	istat(length, "revs in memory");
 	istat(ntlookups, "node trie lookups");
 	istat(ntmisses, "node trie misses");
 	istat(ntrev, "node trie last rev scanned");
 	if (self->ntinitialized) {
 		istat(nt.capacity, "node trie capacity");
 		istat(nt.depth, "node trie depth");
 		istat(nt.length, "node trie count");
 		istat(nt.splits, "node trie splits");
 	}
 
 #undef istat
 
 	return obj;
 
 bail:
 	Py_XDECREF(obj);
 	Py_XDECREF(s);
 	Py_XDECREF(t);
 	return NULL;
 }
 
 /*
  * When we cache a list, we want to be sure the caller can't mutate
  * the cached copy.
  */
 static PyObject *list_copy(PyObject *list)
 {
 	Py_ssize_t len = PyList_GET_SIZE(list);
 	PyObject *newlist = PyList_New(len);
 	Py_ssize_t i;
 
 	if (newlist == NULL)
 		return NULL;
 
 	for (i = 0; i < len; i++) {
 		PyObject *obj = PyList_GET_ITEM(list, i);
 		Py_INCREF(obj);
 		PyList_SET_ITEM(newlist, i, obj);
 	}
 
 	return newlist;
 }
 
 static int check_filter(PyObject *filter, Py_ssize_t arg)
 {
 	if (filter) {
 		PyObject *arglist, *result;
 		int isfiltered;
 
 		arglist = Py_BuildValue("(n)", arg);
 		if (!arglist) {
 			return -1;
 		}
 
 		result = PyObject_Call(filter, arglist, NULL);
 		Py_DECREF(arglist);
 		if (!result) {
 			return -1;
 		}
 
 		/* PyObject_IsTrue returns 1 if true, 0 if false, -1 if error,
 		 * same as this function, so we can just return it directly.*/
 		isfiltered = PyObject_IsTrue(result);
 		Py_DECREF(result);
 		return isfiltered;
 	} else {
 		return 0;
 	}
 }
 
 static inline void set_phase_from_parents(char *phases, int parent_1,
                                           int parent_2, Py_ssize_t i)
 {
 	if (parent_1 >= 0 && phases[parent_1] > phases[i])
 		phases[i] = phases[parent_1];
 	if (parent_2 >= 0 && phases[parent_2] > phases[i])
 		phases[i] = phases[parent_2];
 }
 
 static PyObject *reachableroots2(indexObject *self, PyObject *args)
 {
 
 	/* Input */
 	long minroot;
 	PyObject *includepatharg = NULL;
 	int includepath = 0;
 	/* heads and roots are lists */
 	PyObject *heads = NULL;
 	PyObject *roots = NULL;
 	PyObject *reachable = NULL;
 
 	PyObject *val;
 	Py_ssize_t len = index_length(self);
 	long revnum;
 	Py_ssize_t k;
 	Py_ssize_t i;
 	Py_ssize_t l;
 	int r;
 	int parents[2];
 
 	/* Internal data structure:
 	 * tovisit: array of length len+1 (all revs + nullrev), filled upto
 	 * lentovisit
 	 *
 	 * revstates: array of length len+1 (all revs + nullrev) */
 	int *tovisit = NULL;
 	long lentovisit = 0;
 	enum { RS_SEEN = 1, RS_ROOT = 2, RS_REACHABLE = 4 };
 	char *revstates = NULL;
 
 	/* Get arguments */
 	if (!PyArg_ParseTuple(args, "lO!O!O!", &minroot, &PyList_Type, &heads,
 	                      &PyList_Type, &roots, &PyBool_Type,
 	                      &includepatharg))
 		goto bail;
 
 	if (includepatharg == Py_True)
 		includepath = 1;
 
 	/* Initialize return set */
 	reachable = PyList_New(0);
 	if (reachable == NULL)
 		goto bail;
 
 	/* Initialize internal datastructures */
 	tovisit = (int *)malloc((len + 1) * sizeof(int));
 	if (tovisit == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	revstates = (char *)calloc(len + 1, 1);
 	if (revstates == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	l = PyList_GET_SIZE(roots);
 	for (i = 0; i < l; i++) {
 		revnum = PyInt_AsLong(PyList_GET_ITEM(roots, i));
 		if (revnum == -1 && PyErr_Occurred())
 			goto bail;
 		/* If root is out of range, e.g. wdir(), it must be unreachable
 		 * from heads. So we can just ignore it. */
 		if (revnum + 1 < 0 || revnum + 1 >= len + 1)
 			continue;
 		revstates[revnum + 1] |= RS_ROOT;
 	}
 
 	/* Populate tovisit with all the heads */
 	l = PyList_GET_SIZE(heads);
 	for (i = 0; i < l; i++) {
 		revnum = PyInt_AsLong(PyList_GET_ITEM(heads, i));
 		if (revnum == -1 && PyErr_Occurred())
 			goto bail;
 		if (revnum + 1 < 0 || revnum + 1 >= len + 1) {
 			PyErr_SetString(PyExc_IndexError, "head out of range");
 			goto bail;
 		}
 		if (!(revstates[revnum + 1] & RS_SEEN)) {
 			tovisit[lentovisit++] = (int)revnum;
 			revstates[revnum + 1] |= RS_SEEN;
 		}
 	}
 
 	/* Visit the tovisit list and find the reachable roots */
 	k = 0;
 	while (k < lentovisit) {
 		/* Add the node to reachable if it is a root*/
 		revnum = tovisit[k++];
 		if (revstates[revnum + 1] & RS_ROOT) {
 			revstates[revnum + 1] |= RS_REACHABLE;
 			val = PyInt_FromLong(revnum);
 			if (val == NULL)
 				goto bail;
 			r = PyList_Append(reachable, val);
 			Py_DECREF(val);
 			if (r < 0)
 				goto bail;
 			if (includepath == 0)
 				continue;
 		}
 
 		/* Add its parents to the list of nodes to visit */
 		if (revnum == nullrev)
 			continue;
 		r = index_get_parents(self, revnum, parents, (int)len - 1);
 		if (r < 0)
 			goto bail;
 		for (i = 0; i < 2; i++) {
 			if (!(revstates[parents[i] + 1] & RS_SEEN) &&
 			    parents[i] >= minroot) {
 				tovisit[lentovisit++] = parents[i];
 				revstates[parents[i] + 1] |= RS_SEEN;
 			}
 		}
 	}
 
 	/* Find all the nodes in between the roots we found and the heads
 	 * and add them to the reachable set */
 	if (includepath == 1) {
 		long minidx = minroot;
 		if (minidx < 0)
 			minidx = 0;
 		for (i = minidx; i < len; i++) {
 			if (!(revstates[i + 1] & RS_SEEN))
 				continue;
 			r = index_get_parents(self, i, parents, (int)len - 1);
 			/* Corrupted index file, error is set from
 			 * index_get_parents */
 			if (r < 0)
 				goto bail;
 			if (((revstates[parents[0] + 1] |
 			      revstates[parents[1] + 1]) &
 			     RS_REACHABLE) &&
 			    !(revstates[i + 1] & RS_REACHABLE)) {
 				revstates[i + 1] |= RS_REACHABLE;
 				val = PyInt_FromSsize_t(i);
 				if (val == NULL)
 					goto bail;
 				r = PyList_Append(reachable, val);
 				Py_DECREF(val);
 				if (r < 0)
 					goto bail;
 			}
 		}
 	}
 
 	free(revstates);
 	free(tovisit);
 	return reachable;
 bail:
 	Py_XDECREF(reachable);
 	free(revstates);
 	free(tovisit);
 	return NULL;
 }
 
 static int add_roots_get_min(indexObject *self, PyObject *roots, char *phases,
                              char phase)
 {
 	Py_ssize_t len = index_length(self);
 	PyObject *item;
 	PyObject *iterator;
 	int rev, minrev = -1;
 	char *node;
 
 	if (!PySet_Check(roots)) {
 		PyErr_SetString(PyExc_TypeError,
 		                "roots must be a set of nodes");
 		return -2;
 	}
 	iterator = PyObject_GetIter(roots);
 	if (iterator == NULL)
 		return -2;
 	while ((item = PyIter_Next(iterator))) {
 		if (node_check(self->nodelen, item, &node) == -1)
 			goto failed;
 		rev = index_find_node(self, node);
 		/* null is implicitly public, so negative is invalid */
 		if (rev < 0 || rev >= len)
 			goto failed;
 		phases[rev] = phase;
 		if (minrev == -1 || minrev > rev)
 			minrev = rev;
 		Py_DECREF(item);
 	}
 	Py_DECREF(iterator);
 	return minrev;
 failed:
 	Py_DECREF(iterator);
 	Py_DECREF(item);
 	return -2;
 }
 
 static PyObject *compute_phases_map_sets(indexObject *self, PyObject *args)
 {
 	/* 0: public (untracked), 1: draft, 2: secret, 32: archive,
 	   96: internal */
 	static const char trackedphases[] = {1, 2, 32, 96};
 	PyObject *roots = Py_None;
 	PyObject *phasesetsdict = NULL;
 	PyObject *phasesets[4] = {NULL, NULL, NULL, NULL};
 	Py_ssize_t len = index_length(self);
 	char *phases = NULL;
 	int minphaserev = -1, rev, i;
 	const int numphases = (int)(sizeof(phasesets) / sizeof(phasesets[0]));
 
 	if (!PyArg_ParseTuple(args, "O", &roots))
 		return NULL;
 	if (roots == NULL || !PyDict_Check(roots)) {
 		PyErr_SetString(PyExc_TypeError, "roots must be a dictionary");
 		return NULL;
 	}
 
 	phases = calloc(len, 1);
 	if (phases == NULL) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 
 	for (i = 0; i < numphases; ++i) {
 		PyObject *pyphase = PyInt_FromLong(trackedphases[i]);
 		PyObject *phaseroots = NULL;
 		if (pyphase == NULL)
 			goto release;
 		phaseroots = PyDict_GetItem(roots, pyphase);
 		Py_DECREF(pyphase);
 		if (phaseroots == NULL)
 			continue;
 		rev = add_roots_get_min(self, phaseroots, phases,
 		                        trackedphases[i]);
 		if (rev == -2)
 			goto release;
 		if (rev != -1 && (minphaserev == -1 || rev < minphaserev))
 			minphaserev = rev;
 	}
 
 	for (i = 0; i < numphases; ++i) {
 		phasesets[i] = PySet_New(NULL);
 		if (phasesets[i] == NULL)
 			goto release;
 	}
 
 	if (minphaserev == -1)
 		minphaserev = len;
 	for (rev = minphaserev; rev < len; ++rev) {
 		PyObject *pyphase = NULL;
 		PyObject *pyrev = NULL;
 		int parents[2];
 		/*
 		 * The parent lookup could be skipped for phaseroots, but
 		 * phase --force would historically not recompute them
 		 * correctly, leaving descendents with a lower phase around.
 		 * As such, unconditionally recompute the phase.
 		 */
 		if (index_get_parents(self, rev, parents, (int)len - 1) < 0)
 			goto release;
 		set_phase_from_parents(phases, parents[0], parents[1], rev);
 		switch (phases[rev]) {
 		case 0:
 			continue;
 		case 1:
 			pyphase = phasesets[0];
 			break;
 		case 2:
 			pyphase = phasesets[1];
 			break;
 		case 32:
 			pyphase = phasesets[2];
 			break;
 		case 96:
 			pyphase = phasesets[3];
 			break;
 		default:
 			/* this should never happen since the phase number is
 			 * specified by this function. */
 			PyErr_SetString(PyExc_SystemError,
 			                "bad phase number in internal list");
 			goto release;
 		}
 		pyrev = PyInt_FromLong(rev);
 		if (pyrev == NULL)
 			goto release;
 		if (PySet_Add(pyphase, pyrev) == -1) {
 			Py_DECREF(pyrev);
 			goto release;
 		}
 		Py_DECREF(pyrev);
 	}
 
 	phasesetsdict = _dict_new_presized(numphases);
 	if (phasesetsdict == NULL)
 		goto release;
 	for (i = 0; i < numphases; ++i) {
 		PyObject *pyphase = PyInt_FromLong(trackedphases[i]);
 		if (pyphase == NULL)
 			goto release;
 		if (PyDict_SetItem(phasesetsdict, pyphase, phasesets[i]) ==
 		    -1) {
 			Py_DECREF(pyphase);
 			goto release;
 		}
 		Py_DECREF(phasesets[i]);
 		phasesets[i] = NULL;
 	}
 
 	return Py_BuildValue("nN", len, phasesetsdict);
 
 release:
 	for (i = 0; i < numphases; ++i)
 		Py_XDECREF(phasesets[i]);
 	Py_XDECREF(phasesetsdict);
 
 	free(phases);
 	return NULL;
 }
 
 static PyObject *index_headrevs(indexObject *self, PyObject *args)
 {
 	Py_ssize_t i, j, len;
 	char *nothead = NULL;
 	PyObject *heads = NULL;
 	PyObject *filter = NULL;
 	PyObject *filteredrevs = Py_None;
 
 	if (!PyArg_ParseTuple(args, "|O", &filteredrevs)) {
 		return NULL;
 	}
 
 	if (self->headrevs && filteredrevs == self->filteredrevs)
 		return list_copy(self->headrevs);
 
 	Py_DECREF(self->filteredrevs);
 	self->filteredrevs = filteredrevs;
 	Py_INCREF(filteredrevs);
 
 	if (filteredrevs != Py_None) {
 		filter = PyObject_GetAttrString(filteredrevs, "__contains__");
 		if (!filter) {
 			PyErr_SetString(
 			    PyExc_TypeError,
 			    "filteredrevs has no attribute __contains__");
 			goto bail;
 		}
 	}
 
 	len = index_length(self);
 	heads = PyList_New(0);
 	if (heads == NULL)
 		goto bail;
 	if (len == 0) {
 		PyObject *nullid = PyInt_FromLong(-1);
 		if (nullid == NULL || PyList_Append(heads, nullid) == -1) {
 			Py_XDECREF(nullid);
 			goto bail;
 		}
 		goto done;
 	}
 
 	nothead = calloc(len, 1);
 	if (nothead == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	for (i = len - 1; i >= 0; i--) {
 		int isfiltered;
 		int parents[2];
 
 		/* If nothead[i] == 1, it means we've seen an unfiltered child
 		 * of this node already, and therefore this node is not
 		 * filtered. So we can skip the expensive check_filter step.
 		 */
 		if (nothead[i] != 1) {
 			isfiltered = check_filter(filter, i);
 			if (isfiltered == -1) {
 				PyErr_SetString(PyExc_TypeError,
 				                "unable to check filter");
 				goto bail;
 			}
 
 			if (isfiltered) {
 				nothead[i] = 1;
 				continue;
 			}
 		}
 
 		if (index_get_parents(self, i, parents, (int)len - 1) < 0)
 			goto bail;
 		for (j = 0; j < 2; j++) {
 			if (parents[j] >= 0)
 				nothead[parents[j]] = 1;
 		}
 	}
 
 	for (i = 0; i < len; i++) {
 		PyObject *head;
 
 		if (nothead[i])
 			continue;
 		head = PyInt_FromSsize_t(i);
 		if (head == NULL || PyList_Append(heads, head) == -1) {
 			Py_XDECREF(head);
 			goto bail;
 		}
 	}
 
 done:
 	self->headrevs = heads;
 	Py_XDECREF(filter);
 	free(nothead);
 	return list_copy(self->headrevs);
 bail:
 	Py_XDECREF(filter);
 	Py_XDECREF(heads);
 	free(nothead);
 	return NULL;
 }
 
 /**
  * Obtain the base revision index entry.
  *
  * Callers must ensure that rev >= 0 or illegal memory access may occur.
  */
 static inline int index_baserev(indexObject *self, int rev)
 {
 	const char *data;
 	int result;
 
 	data = index_deref(self, rev);
 	if (data == NULL)
 		return -2;
 	result = getbe32(data + 16);
 
 	if (result > rev) {
 		PyErr_Format(
 		    PyExc_ValueError,
 		    "corrupted revlog, revision base above revision: %d, %d",
 		    rev, result);
 		return -2;
 	}
 	if (result < -1) {
 		PyErr_Format(
 		    PyExc_ValueError,
 		    "corrupted revlog, revision base out of range: %d, %d", rev,
 		    result);
 		return -2;
 	}
 	return result;
 }
 
 /**
  * Find if a revision is a snapshot or not
  *
  * Only relevant for sparse-revlog case.
  * Callers must ensure that rev is in a valid range.
  */
 static int index_issnapshotrev(indexObject *self, Py_ssize_t rev)
 {
 	int ps[2];
 	Py_ssize_t base;
 	while (rev >= 0) {
 		base = (Py_ssize_t)index_baserev(self, rev);
 		if (base == rev) {
 			base = -1;
 		}
 		if (base == -2) {
 			assert(PyErr_Occurred());
 			return -1;
 		}
 		if (base == -1) {
 			return 1;
 		}
 		if (index_get_parents(self, rev, ps, (int)rev) < 0) {
 			assert(PyErr_Occurred());
 			return -1;
 		};
 		if (base == ps[0] || base == ps[1]) {
 			return 0;
 		}
 		rev = base;
 	}
 	return rev == -1;
 }
 
 static PyObject *index_issnapshot(indexObject *self, PyObject *value)
 {
 	long rev;
 	int issnap;
 	Py_ssize_t length = index_length(self);
 
 	if (!pylong_to_long(value, &rev)) {
 		return NULL;
 	}
 	if (rev < -1 || rev >= length) {
 		PyErr_Format(PyExc_ValueError, "revlog index out of range: %ld",
 		             rev);
 		return NULL;
 	};
 	issnap = index_issnapshotrev(self, (Py_ssize_t)rev);
 	if (issnap < 0) {
 		return NULL;
 	};
 	return PyBool_FromLong((long)issnap);
 }
 
 static PyObject *index_findsnapshots(indexObject *self, PyObject *args)
 {
 	Py_ssize_t start_rev;
 	PyObject *cache;
 	Py_ssize_t base;
 	Py_ssize_t rev;
 	PyObject *key = NULL;
 	PyObject *value = NULL;
 	const Py_ssize_t length = index_length(self);
 	if (!PyArg_ParseTuple(args, "O!n", &PyDict_Type, &cache, &start_rev)) {
 		return NULL;
 	}
 	for (rev = start_rev; rev < length; rev++) {
 		int issnap;
 		PyObject *allvalues = NULL;
 		issnap = index_issnapshotrev(self, rev);
 		if (issnap < 0) {
 			goto bail;
 		}
 		if (issnap == 0) {
 			continue;
 		}
 		base = (Py_ssize_t)index_baserev(self, rev);
 		if (base == rev) {
 			base = -1;
 		}
 		if (base == -2) {
 			assert(PyErr_Occurred());
 			goto bail;
 		}
 		key = PyInt_FromSsize_t(base);
 		allvalues = PyDict_GetItem(cache, key);
 		if (allvalues == NULL && PyErr_Occurred()) {
 			goto bail;
 		}
 		if (allvalues == NULL) {
 			int r;
 			allvalues = PyList_New(0);
 			if (!allvalues) {
 				goto bail;
 			}
 			r = PyDict_SetItem(cache, key, allvalues);
 			Py_DECREF(allvalues);
 			if (r < 0) {
 				goto bail;
 			}
 		}
 		value = PyInt_FromSsize_t(rev);
 		if (PyList_Append(allvalues, value)) {
 			goto bail;
 		}
 		Py_CLEAR(key);
 		Py_CLEAR(value);
 	}
 	Py_RETURN_NONE;
 bail:
 	Py_XDECREF(key);
 	Py_XDECREF(value);
 	return NULL;
 }
 
 static PyObject *index_deltachain(indexObject *self, PyObject *args)
 {
 	int rev, generaldelta;
 	PyObject *stoparg;
 	int stoprev, iterrev, baserev = -1;
 	int stopped;
 	PyObject *chain = NULL, *result = NULL;
 	const Py_ssize_t length = index_length(self);
 
 	if (!PyArg_ParseTuple(args, "iOi", &rev, &stoparg, &generaldelta)) {
 		return NULL;
 	}
 
 	if (PyInt_Check(stoparg)) {
 		stoprev = (int)PyInt_AsLong(stoparg);
 		if (stoprev == -1 && PyErr_Occurred()) {
 			return NULL;
 		}
 	} else if (stoparg == Py_None) {
 		stoprev = -2;
 	} else {
 		PyErr_SetString(PyExc_ValueError,
 		                "stoprev must be integer or None");
 		return NULL;
 	}
 
 	if (rev < 0 || rev >= length) {
 		PyErr_SetString(PyExc_ValueError, "revlog index out of range");
 		return NULL;
 	}
 
 	chain = PyList_New(0);
 	if (chain == NULL) {
 		return NULL;
 	}
 
 	baserev = index_baserev(self, rev);
 
 	/* This should never happen. */
 	if (baserev <= -2) {
 		/* Error should be set by index_deref() */
 		assert(PyErr_Occurred());
 		goto bail;
 	}
 
 	iterrev = rev;
 
 	while (iterrev != baserev && iterrev != stoprev) {
 		PyObject *value = PyInt_FromLong(iterrev);
 		if (value == NULL) {
 			goto bail;
 		}
 		if (PyList_Append(chain, value)) {
 			Py_DECREF(value);
 			goto bail;
 		}
 		Py_DECREF(value);
 
 		if (generaldelta) {
 			iterrev = baserev;
 		} else {
 			iterrev--;
 		}
 
 		if (iterrev < 0) {
 			break;
 		}
 
 		if (iterrev >= length) {
 			PyErr_SetString(PyExc_IndexError,
 			                "revision outside index");
 			return NULL;
 		}
 
 		baserev = index_baserev(self, iterrev);
 
 		/* This should never happen. */
 		if (baserev <= -2) {
 			/* Error should be set by index_deref() */
 			assert(PyErr_Occurred());
 			goto bail;
 		}
 	}
 
 	if (iterrev == stoprev) {
 		stopped = 1;
 	} else {
 		PyObject *value = PyInt_FromLong(iterrev);
 		if (value == NULL) {
 			goto bail;
 		}
 		if (PyList_Append(chain, value)) {
 			Py_DECREF(value);
 			goto bail;
 		}
 		Py_DECREF(value);
 
 		stopped = 0;
 	}
 
 	if (PyList_Reverse(chain)) {
 		goto bail;
 	}
 
 	result = Py_BuildValue("OO", chain, stopped ? Py_True : Py_False);
 	Py_DECREF(chain);
 	return result;
 
 bail:
 	Py_DECREF(chain);
 	return NULL;
 }
 
 static inline int64_t
 index_segment_span(indexObject *self, Py_ssize_t start_rev, Py_ssize_t end_rev)
 {
 	int64_t start_offset;
 	int64_t end_offset;
 	int end_size;
 	start_offset = index_get_start(self, start_rev);
 	if (start_offset < 0) {
 		return -1;
 	}
 	end_offset = index_get_start(self, end_rev);
 	if (end_offset < 0) {
 		return -1;
 	}
 	end_size = index_get_length(self, end_rev);
 	if (end_size < 0) {
 		return -1;
 	}
 	if (end_offset < start_offset) {
 		PyErr_Format(PyExc_ValueError,
 		             "corrupted revlog index: inconsistent offset "
 		             "between revisions (%zd) and (%zd)",
 		             start_rev, end_rev);
 		return -1;
 	}
 	return (end_offset - start_offset) + (int64_t)end_size;
 }
 
 /* returns endidx so that revs[startidx:endidx] has no empty trailing revs */
 static Py_ssize_t trim_endidx(indexObject *self, const Py_ssize_t *revs,
                               Py_ssize_t startidx, Py_ssize_t endidx)
 {
 	int length;
 	while (endidx > 1 && endidx > startidx) {
 		length = index_get_length(self, revs[endidx - 1]);
 		if (length < 0) {
 			return -1;
 		}
 		if (length != 0) {
 			break;
 		}
 		endidx -= 1;
 	}
 	return endidx;
 }
 
 struct Gap {
 	int64_t size;
 	Py_ssize_t idx;
 };
 
 static int gap_compare(const void *left, const void *right)
 {
 	const struct Gap *l_left = ((const struct Gap *)left);
 	const struct Gap *l_right = ((const struct Gap *)right);
 	if (l_left->size < l_right->size) {
 		return -1;
 	} else if (l_left->size > l_right->size) {
 		return 1;
 	}
 	return 0;
 }
 static int Py_ssize_t_compare(const void *left, const void *right)
 {
 	const Py_ssize_t l_left = *(const Py_ssize_t *)left;
 	const Py_ssize_t l_right = *(const Py_ssize_t *)right;
 	if (l_left < l_right) {
 		return -1;
 	} else if (l_left > l_right) {
 		return 1;
 	}
 	return 0;
 }
 
 static PyObject *index_slicechunktodensity(indexObject *self, PyObject *args)
 {
 	/* method arguments */
 	PyObject *list_revs = NULL; /* revisions in the chain */
 	double targetdensity = 0;   /* min density to achieve */
 	Py_ssize_t mingapsize = 0;  /* threshold to ignore gaps */
 
 	/* other core variables */
 	Py_ssize_t idxlen = index_length(self);
 	Py_ssize_t i;            /* used for various iteration */
 	PyObject *result = NULL; /* the final return of the function */
 
 	/* generic information about the delta chain being slice */
 	Py_ssize_t num_revs = 0;    /* size of the full delta chain */
 	Py_ssize_t *revs = NULL;    /* native array of revision in the chain */
 	int64_t chainpayload = 0;   /* sum of all delta in the chain */
 	int64_t deltachainspan = 0; /* distance from first byte to last byte */
 
 	/* variable used for slicing the delta chain */
 	int64_t readdata = 0; /* amount of data currently planned to be read */
 	double density = 0;   /* ration of payload data compared to read ones */
 	int64_t previous_end;
 	struct Gap *gaps = NULL; /* array of notable gap in the chain */
 	Py_ssize_t num_gaps =
 	    0; /* total number of notable gap recorded so far */
 	Py_ssize_t *selected_indices = NULL; /* indices of gap skipped over */
 	Py_ssize_t num_selected = 0;         /* number of gaps skipped */
 	PyObject *chunk = NULL;              /* individual slice */
 	PyObject *allchunks = NULL;          /* all slices */
 	Py_ssize_t previdx;
 
 	/* parsing argument */
 	if (!PyArg_ParseTuple(args, "O!dn", &PyList_Type, &list_revs,
 	                      &targetdensity, &mingapsize)) {
 		goto bail;
 	}
 
 	/* If the delta chain contains a single element, we do not need slicing
 	 */
 	num_revs = PyList_GET_SIZE(list_revs);
 	if (num_revs <= 1) {
 		result = PyTuple_Pack(1, list_revs);
 		goto done;
 	}
 
 	/* Turn the python list into a native integer array (for efficiency) */
 	revs = (Py_ssize_t *)calloc(num_revs, sizeof(Py_ssize_t));
 	if (revs == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 	for (i = 0; i < num_revs; i++) {
 		Py_ssize_t revnum = PyInt_AsLong(PyList_GET_ITEM(list_revs, i));
 		if (revnum == -1 && PyErr_Occurred()) {
 			goto bail;
 		}
 		if (revnum < nullrev || revnum >= idxlen) {
 			PyErr_Format(PyExc_IndexError,
 			             "index out of range: %zd", revnum);
 			goto bail;
 		}
 		revs[i] = revnum;
 	}
 
 	/* Compute and check various property of the unsliced delta chain */
 	deltachainspan = index_segment_span(self, revs[0], revs[num_revs - 1]);
 	if (deltachainspan < 0) {
 		goto bail;
 	}
 
 	if (deltachainspan <= mingapsize) {
 		result = PyTuple_Pack(1, list_revs);
 		goto done;
 	}
 	chainpayload = 0;
 	for (i = 0; i < num_revs; i++) {
 		int tmp = index_get_length(self, revs[i]);
 		if (tmp < 0) {
 			goto bail;
 		}
 		chainpayload += tmp;
 	}
 
 	readdata = deltachainspan;
 	density = 1.0;
 
 	if (0 < deltachainspan) {
 		density = (double)chainpayload / (double)deltachainspan;
 	}
 
 	if (density >= targetdensity) {
 		result = PyTuple_Pack(1, list_revs);
 		goto done;
 	}
 
 	/* if chain is too sparse, look for relevant gaps */
 	gaps = (struct Gap *)calloc(num_revs, sizeof(struct Gap));
 	if (gaps == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	previous_end = -1;
 	for (i = 0; i < num_revs; i++) {
 		int64_t revstart;
 		int revsize;
 		revstart = index_get_start(self, revs[i]);
 		if (revstart < 0) {
 			goto bail;
 		};
 		revsize = index_get_length(self, revs[i]);
 		if (revsize < 0) {
 			goto bail;
 		};
 		if (revsize == 0) {
 			continue;
 		}
 		if (previous_end >= 0) {
 			int64_t gapsize = revstart - previous_end;
 			if (gapsize > mingapsize) {
 				gaps[num_gaps].size = gapsize;
 				gaps[num_gaps].idx = i;
 				num_gaps += 1;
 			}
 		}
 		previous_end = revstart + revsize;
 	}
 	if (num_gaps == 0) {
 		result = PyTuple_Pack(1, list_revs);
 		goto done;
 	}
 	qsort(gaps, num_gaps, sizeof(struct Gap), &gap_compare);
 
 	/* Slice the largest gap first, they improve the density the most */
 	selected_indices =
 	    (Py_ssize_t *)malloc((num_gaps + 1) * sizeof(Py_ssize_t));
 	if (selected_indices == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	for (i = num_gaps - 1; i >= 0; i--) {
 		selected_indices[num_selected] = gaps[i].idx;
 		readdata -= gaps[i].size;
 		num_selected += 1;
 		if (readdata <= 0) {
 			density = 1.0;
 		} else {
 			density = (double)chainpayload / (double)readdata;
 		}
 		if (density >= targetdensity) {
 			break;
 		}
 	}
 	qsort(selected_indices, num_selected, sizeof(Py_ssize_t),
 	      &Py_ssize_t_compare);
 
 	/* create the resulting slice */
 	allchunks = PyList_New(0);
 	if (allchunks == NULL) {
 		goto bail;
 	}
 	previdx = 0;
 	selected_indices[num_selected] = num_revs;
 	for (i = 0; i <= num_selected; i++) {
 		Py_ssize_t idx = selected_indices[i];
 		Py_ssize_t endidx = trim_endidx(self, revs, previdx, idx);
 		if (endidx < 0) {
 			goto bail;
 		}
 		if (previdx < endidx) {
 			chunk = PyList_GetSlice(list_revs, previdx, endidx);
 			if (chunk == NULL) {
 				goto bail;
 			}
 			if (PyList_Append(allchunks, chunk) == -1) {
 				goto bail;
 			}
 			Py_DECREF(chunk);
 			chunk = NULL;
 		}
 		previdx = idx;
 	}
 	result = allchunks;
 	goto done;
 
 bail:
 	Py_XDECREF(allchunks);
 	Py_XDECREF(chunk);
 done:
 	free(revs);
 	free(gaps);
 	free(selected_indices);
 	return result;
 }
 
 static inline int nt_level(const char *node, Py_ssize_t level)
 {
 	int v = node[level >> 1];
 	if (!(level & 1))
 		v >>= 4;
 	return v & 0xf;
 }
 
 /*
  * Return values:
  *
  *   -4: match is ambiguous (multiple candidates)
  *   -2: not found
  * rest: valid rev
  */
 static int nt_find(nodetree *self, const char *node, Py_ssize_t nodelen,
                    int hex)
 {
 	int (*getnybble)(const char *, Py_ssize_t) = hex ? hexdigit : nt_level;
 	int level, maxlevel, off;
 
 	/* If the input is binary, do a fast check for the nullid first. */
 	if (!hex && nodelen == self->nodelen && node[0] == '\0' &&
 	    node[1] == '\0' && memcmp(node, nullid, self->nodelen) == 0)
 		return -1;
 
 	if (hex)
 		maxlevel = nodelen;
 	else
 		maxlevel = 2 * nodelen;
 	if (maxlevel > 2 * self->nodelen)
 		maxlevel = 2 * self->nodelen;
 
 	for (level = off = 0; level < maxlevel; level++) {
 		int k = getnybble(node, level);
 		nodetreenode *n = &self->nodes[off];
 		int v = n->children[k];
 
 		if (v < 0) {
 			const char *n;
 			Py_ssize_t i;
 
 			v = -(v + 2);
 			n = index_node(self->index, v);
 			if (n == NULL)
 				return -2;
 			for (i = level; i < maxlevel; i++)
 				if (getnybble(node, i) != nt_level(n, i))
 					return -2;
 			return v;
 		}
 		if (v == 0)
 			return -2;
 		off = v;
 	}
 	/* multiple matches against an ambiguous prefix */
 	return -4;
 }
 
 static int nt_new(nodetree *self)
 {
 	if (self->length == self->capacity) {
 		size_t newcapacity;
 		nodetreenode *newnodes;
 		newcapacity = self->capacity * 2;
 		if (newcapacity >= SIZE_MAX / sizeof(nodetreenode)) {
 			PyErr_SetString(PyExc_MemoryError,
 			                "overflow in nt_new");
 			return -1;
 		}
 		newnodes =
 		    realloc(self->nodes, newcapacity * sizeof(nodetreenode));
 		if (newnodes == NULL) {
 			PyErr_SetString(PyExc_MemoryError, "out of memory");
 			return -1;
 		}
 		self->capacity = newcapacity;
 		self->nodes = newnodes;
 		memset(&self->nodes[self->length], 0,
 		       sizeof(nodetreenode) * (self->capacity - self->length));
 	}
 	return self->length++;
 }
 
 static int nt_insert(nodetree *self, const char *node, int rev)
 {
 	int level = 0;
 	int off = 0;
 
 	while (level < 2 * self->nodelen) {
 		int k = nt_level(node, level);
 		nodetreenode *n;
 		int v;
 
 		n = &self->nodes[off];
 		v = n->children[k];
 
 		if (v == 0) {
 			n->children[k] = -rev - 2;
 			return 0;
 		}
 		if (v < 0) {
 			const char *oldnode =
 			    index_node_existing(self->index, -(v + 2));
 			int noff;
 
 			if (oldnode == NULL)
 				return -1;
 			if (!memcmp(oldnode, node, self->nodelen)) {
 				n->children[k] = -rev - 2;
 				return 0;
 			}
 			noff = nt_new(self);
 			if (noff == -1)
 				return -1;
 			/* self->nodes may have been changed by realloc */
 			self->nodes[off].children[k] = noff;
 			off = noff;
 			n = &self->nodes[off];
 			n->children[nt_level(oldnode, ++level)] = v;
 			if (level > self->depth)
 				self->depth = level;
 			self->splits += 1;
 		} else {
 			level += 1;
 			off = v;
 		}
 	}
 
 	return -1;
 }
 
 static PyObject *ntobj_insert(nodetreeObject *self, PyObject *args)
 {
 	Py_ssize_t rev;
 	const char *node;
 	Py_ssize_t length;
 	if (!PyArg_ParseTuple(args, "n", &rev))
 		return NULL;
 	length = index_length(self->nt.index);
 	if (rev < 0 || rev >= length) {
 		PyErr_SetString(PyExc_ValueError, "revlog index out of range");
 		return NULL;
 	}
 	node = index_node_existing(self->nt.index, rev);
 	if (nt_insert(&self->nt, node, (int)rev) == -1)
 		return NULL;
 	Py_RETURN_NONE;
 }
 
 static int nt_delete_node(nodetree *self, const char *node)
 {
 	/* rev==-2 happens to get encoded as 0, which is interpreted as not set
 	 */
 	return nt_insert(self, node, -2);
 }
 
 static int nt_init(nodetree *self, indexObject *index, unsigned capacity)
 {
 	/* Initialize before overflow-checking to avoid nt_dealloc() crash. */
 	self->nodes = NULL;
 
 	self->index = index;
 	/* The input capacity is in terms of revisions, while the field is in
 	 * terms of nodetree nodes. */
 	self->capacity = (capacity < 4 ? 4 : capacity / 2);
 	self->nodelen = index->nodelen;
 	self->depth = 0;
 	self->splits = 0;
 	if (self->capacity > SIZE_MAX / sizeof(nodetreenode)) {
 		PyErr_SetString(PyExc_ValueError, "overflow in init_nt");
 		return -1;
 	}
 	self->nodes = calloc(self->capacity, sizeof(nodetreenode));
 	if (self->nodes == NULL) {
 		PyErr_NoMemory();
 		return -1;
 	}
 	self->length = 1;
 	return 0;
 }
 
 static int ntobj_init(nodetreeObject *self, PyObject *args)
 {
 	PyObject *index;
 	unsigned capacity;
 	if (!PyArg_ParseTuple(args, "O!I", &HgRevlogIndex_Type, &index,
 	                      &capacity))
 		return -1;
 	Py_INCREF(index);
 	return nt_init(&self->nt, (indexObject *)index, capacity);
 }
 
 static int nt_partialmatch(nodetree *self, const char *node, Py_ssize_t nodelen)
 {
 	return nt_find(self, node, nodelen, 1);
 }
 
 /*
  * Find the length of the shortest unique prefix of node.
  *
  * Return values:
  *
  *   -3: error (exception set)
  *   -2: not found (no exception set)
  * rest: length of shortest prefix
  */
 static int nt_shortest(nodetree *self, const char *node)
 {
 	int level, off;
 
 	for (level = off = 0; level < 2 * self->nodelen; level++) {
 		int k, v;
 		nodetreenode *n = &self->nodes[off];
 		k = nt_level(node, level);
 		v = n->children[k];
 		if (v < 0) {
 			const char *n;
 			v = -(v + 2);
 			n = index_node_existing(self->index, v);
 			if (n == NULL)
 				return -3;
 			if (memcmp(node, n, self->nodelen) != 0)
 				/*
 				 * Found a unique prefix, but it wasn't for the
 				 * requested node (i.e the requested node does
 				 * not exist).
 				 */
 				return -2;
 			return level + 1;
 		}
 		if (v == 0)
 			return -2;
 		off = v;
 	}
 	/*
 	 * The node was still not unique after 40 hex digits, so this won't
 	 * happen. Also, if we get here, then there's a programming error in
 	 * this file that made us insert a node longer than 40 hex digits.
 	 */
 	PyErr_SetString(PyExc_Exception, "broken node tree");
 	return -3;
 }
 
 static PyObject *ntobj_shortest(nodetreeObject *self, PyObject *args)
 {
 	PyObject *val;
 	char *node;
 	int length;
 
 	if (!PyArg_ParseTuple(args, "O", &val))
 		return NULL;
 	if (node_check(self->nt.nodelen, val, &node) == -1)
 		return NULL;
 
 	length = nt_shortest(&self->nt, node);
 	if (length == -3)
 		return NULL;
 	if (length == -2) {
 		raise_revlog_error();
 		return NULL;
 	}
 	return PyInt_FromLong(length);
 }
 
 static void nt_dealloc(nodetree *self)
 {
 	free(self->nodes);
 	self->nodes = NULL;
 }
 
 static void ntobj_dealloc(nodetreeObject *self)
 {
 	Py_XDECREF(self->nt.index);
 	nt_dealloc(&self->nt);
 	PyObject_Del(self);
 }
 
 static PyMethodDef ntobj_methods[] = {
     {"insert", (PyCFunction)ntobj_insert, METH_VARARGS,
      "insert an index entry"},
     {"shortest", (PyCFunction)ntobj_shortest, METH_VARARGS,
      "find length of shortest hex nodeid of a binary ID"},
     {NULL} /* Sentinel */
 };
 
 static PyTypeObject nodetreeType = {
     PyVarObject_HEAD_INIT(NULL, 0) /* header */
     "parsers.nodetree",            /* tp_name */
     sizeof(nodetreeObject),        /* tp_basicsize */
     0,                             /* tp_itemsize */
     (destructor)ntobj_dealloc,     /* tp_dealloc */
     0,                             /* tp_print */
     0,                             /* tp_getattr */
     0,                             /* tp_setattr */
     0,                             /* tp_compare */
     0,                             /* tp_repr */
     0,                             /* tp_as_number */
     0,                             /* tp_as_sequence */
     0,                             /* tp_as_mapping */
     0,                             /* tp_hash */
     0,                             /* tp_call */
     0,                             /* tp_str */
     0,                             /* tp_getattro */
     0,                             /* tp_setattro */
     0,                             /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,            /* tp_flags */
     "nodetree",                    /* tp_doc */
     0,                             /* tp_traverse */
     0,                             /* tp_clear */
     0,                             /* tp_richcompare */
     0,                             /* tp_weaklistoffset */
     0,                             /* tp_iter */
     0,                             /* tp_iternext */
     ntobj_methods,                 /* tp_methods */
     0,                             /* tp_members */
     0,                             /* tp_getset */
     0,                             /* tp_base */
     0,                             /* tp_dict */
     0,                             /* tp_descr_get */
     0,                             /* tp_descr_set */
     0,                             /* tp_dictoffset */
     (initproc)ntobj_init,          /* tp_init */
     0,                             /* tp_alloc */
 };
 
 static int index_init_nt(indexObject *self)
 {
 	if (!self->ntinitialized) {
 		if (nt_init(&self->nt, self, (int)self->length) == -1) {
 			nt_dealloc(&self->nt);
 			return -1;
 		}
 		if (nt_insert(&self->nt, nullid, -1) == -1) {
 			nt_dealloc(&self->nt);
 			return -1;
 		}
 		self->ntinitialized = 1;
 		self->ntrev = (int)index_length(self);
 		self->ntlookups = 1;
 		self->ntmisses = 0;
 	}
 	return 0;
 }
 
 /*
  * Return values:
  *
  *   -3: error (exception set)
  *   -2: not found (no exception set)
  * rest: valid rev
  */
 static int index_find_node(indexObject *self, const char *node)
 {
 	int rev;
 
 	if (index_init_nt(self) == -1)
 		return -3;
 
 	self->ntlookups++;
 	rev = nt_find(&self->nt, node, self->nodelen, 0);
 	if (rev >= -1)
 		return rev;
 
 	/*
 	 * For the first handful of lookups, we scan the entire index,
 	 * and cache only the matching nodes. This optimizes for cases
 	 * like "hg tip", where only a few nodes are accessed.
 	 *
 	 * After that, we cache every node we visit, using a single
 	 * scan amortized over multiple lookups.  This gives the best
 	 * bulk performance, e.g. for "hg log".
 	 */
 	if (self->ntmisses++ < 4) {
 		for (rev = self->ntrev - 1; rev >= 0; rev--) {
 			const char *n = index_node_existing(self, rev);
 			if (n == NULL)
 				return -3;
 			if (memcmp(node, n, self->nodelen) == 0) {
 				if (nt_insert(&self->nt, n, rev) == -1)
 					return -3;
 				break;
 			}
 		}
 	} else {
 		for (rev = self->ntrev - 1; rev >= 0; rev--) {
 			const char *n = index_node_existing(self, rev);
 			if (n == NULL)
 				return -3;
 			if (nt_insert(&self->nt, n, rev) == -1) {
 				self->ntrev = rev + 1;
 				return -3;
 			}
 			if (memcmp(node, n, self->nodelen) == 0) {
 				break;
 			}
 		}
 		self->ntrev = rev;
 	}
 
 	if (rev >= 0)
 		return rev;
 	return -2;
 }
 
 static PyObject *index_getitem(indexObject *self, PyObject *value)
 {
 	char *node;
 	int rev;
 
 	if (PyInt_Check(value)) {
 		long idx;
 		if (!pylong_to_long(value, &idx)) {
 			return NULL;
 		}
 		return index_get(self, idx);
 	}
 
 	if (node_check(self->nodelen, value, &node) == -1)
 		return NULL;
 	rev = index_find_node(self, node);
 	if (rev >= -1)
 		return PyInt_FromLong(rev);
 	if (rev == -2)
 		raise_revlog_error();
 	return NULL;
 }
 
 /*
  * Fully populate the radix tree.
  */
 static int index_populate_nt(indexObject *self)
 {
 	int rev;
 	if (self->ntrev > 0) {
 		for (rev = self->ntrev - 1; rev >= 0; rev--) {
 			const char *n = index_node_existing(self, rev);
 			if (n == NULL)
 				return -1;
 			if (nt_insert(&self->nt, n, rev) == -1)
 				return -1;
 		}
 		self->ntrev = -1;
 	}
 	return 0;
 }
 
 static PyObject *index_partialmatch(indexObject *self, PyObject *args)
 {
 	const char *fullnode;
 	Py_ssize_t nodelen;
 	char *node;
 	int rev, i;
 
 	if (!PyArg_ParseTuple(args, PY23("s#", "y#"), &node, &nodelen))
 		return NULL;
 
 	if (nodelen < 1) {
 		PyErr_SetString(PyExc_ValueError, "key too short");
 		return NULL;
 	}
 
 	if (nodelen > 2 * self->nodelen) {
 		PyErr_SetString(PyExc_ValueError, "key too long");
 		return NULL;
 	}
 
 	for (i = 0; i < nodelen; i++)
 		hexdigit(node, i);
 	if (PyErr_Occurred()) {
 		/* input contains non-hex characters */
 		PyErr_Clear();
 		Py_RETURN_NONE;
 	}
 
 	if (index_init_nt(self) == -1)
 		return NULL;
 	if (index_populate_nt(self) == -1)
 		return NULL;
 	rev = nt_partialmatch(&self->nt, node, nodelen);
 
 	switch (rev) {
 	case -4:
 		raise_revlog_error();
 		return NULL;
 	case -2:
 		Py_RETURN_NONE;
 	case -1:
 		return PyBytes_FromStringAndSize(nullid, self->nodelen);
 	}
 
 	fullnode = index_node_existing(self, rev);
 	if (fullnode == NULL) {
 		return NULL;
 	}
 	return PyBytes_FromStringAndSize(fullnode, self->nodelen);
 }
 
 static PyObject *index_shortest(indexObject *self, PyObject *args)
 {
 	PyObject *val;
 	char *node;
 	int length;
 
 	if (!PyArg_ParseTuple(args, "O", &val))
 		return NULL;
 	if (node_check(self->nodelen, val, &node) == -1)
 		return NULL;
 
 	self->ntlookups++;
 	if (index_init_nt(self) == -1)
 		return NULL;
 	if (index_populate_nt(self) == -1)
 		return NULL;
 	length = nt_shortest(&self->nt, node);
 	if (length == -3)
 		return NULL;
 	if (length == -2) {
 		raise_revlog_error();
 		return NULL;
 	}
 	return PyInt_FromLong(length);
 }
 
 static PyObject *index_m_get(indexObject *self, PyObject *args)
 {
 	PyObject *val;
 	char *node;
 	int rev;
 
 	if (!PyArg_ParseTuple(args, "O", &val))
 		return NULL;
 	if (node_check(self->nodelen, val, &node) == -1)
 		return NULL;
 	rev = index_find_node(self, node);
 	if (rev == -3)
 		return NULL;
 	if (rev == -2)
 		Py_RETURN_NONE;
 	return PyInt_FromLong(rev);
 }
 
 static int index_contains(indexObject *self, PyObject *value)
 {
 	char *node;
 
 	if (PyInt_Check(value)) {
 		long rev;
 		if (!pylong_to_long(value, &rev)) {
 			return -1;
 		}
 		return rev >= -1 && rev < index_length(self);
 	}
 
 	if (node_check(self->nodelen, value, &node) == -1)
 		return -1;
 
 	switch (index_find_node(self, node)) {
 	case -3:
 		return -1;
 	case -2:
 		return 0;
 	default:
 		return 1;
 	}
 }
 
 static PyObject *index_m_has_node(indexObject *self, PyObject *args)
 {
 	int ret = index_contains(self, args);
 	if (ret < 0)
 		return NULL;
 	return PyBool_FromLong((long)ret);
 }
 
 static PyObject *index_m_rev(indexObject *self, PyObject *val)
 {
 	char *node;
 	int rev;
 
 	if (node_check(self->nodelen, val, &node) == -1)
 		return NULL;
 	rev = index_find_node(self, node);
 	if (rev >= -1)
 		return PyInt_FromLong(rev);
 	if (rev == -2)
 		raise_revlog_error();
 	return NULL;
 }
 
 typedef uint64_t bitmask;
 
 /*
  * Given a disjoint set of revs, return all candidates for the
  * greatest common ancestor. In revset notation, this is the set
  * "heads(::a and ::b and ...)"
  */
 static PyObject *find_gca_candidates(indexObject *self, const int *revs,
                                      int revcount)
 {
 	const bitmask allseen = (1ull << revcount) - 1;
 	const bitmask poison = 1ull << revcount;
 	PyObject *gca = PyList_New(0);
 	int i, v, interesting;
 	int maxrev = -1;
 	bitmask sp;
 	bitmask *seen;
 
 	if (gca == NULL)
 		return PyErr_NoMemory();
 
 	for (i = 0; i < revcount; i++) {
 		if (revs[i] > maxrev)
 			maxrev = revs[i];
 	}
 
 	seen = calloc(sizeof(*seen), maxrev + 1);
 	if (seen == NULL) {
 		Py_DECREF(gca);
 		return PyErr_NoMemory();
 	}
 
 	for (i = 0; i < revcount; i++)
 		seen[revs[i]] = 1ull << i;
 
 	interesting = revcount;
 
 	for (v = maxrev; v >= 0 && interesting; v--) {
 		bitmask sv = seen[v];
 		int parents[2];
 
 		if (!sv)
 			continue;
 
 		if (sv < poison) {
 			interesting -= 1;
 			if (sv == allseen) {
 				PyObject *obj = PyInt_FromLong(v);
 				if (obj == NULL)
 					goto bail;
 				if (PyList_Append(gca, obj) == -1) {
 					Py_DECREF(obj);
 					goto bail;
 				}
 				sv |= poison;
 				for (i = 0; i < revcount; i++) {
 					if (revs[i] == v)
 						goto done;
 				}
 			}
 		}
 		if (index_get_parents(self, v, parents, maxrev) < 0)
 			goto bail;
 
 		for (i = 0; i < 2; i++) {
 			int p = parents[i];
 			if (p == -1)
 				continue;
 			sp = seen[p];
 			if (sv < poison) {
 				if (sp == 0) {
 					seen[p] = sv;
 					interesting++;
 				} else if (sp != sv)
 					seen[p] |= sv;
 			} else {
 				if (sp && sp < poison)
 					interesting--;
 				seen[p] = sv;
 			}
 		}
 	}
 
 done:
 	free(seen);
 	return gca;
 bail:
 	free(seen);
 	Py_XDECREF(gca);
 	return NULL;
 }
 
 /*
  * Given a disjoint set of revs, return the subset with the longest
  * path to the root.
  */
 static PyObject *find_deepest(indexObject *self, PyObject *revs)
 {
 	const Py_ssize_t revcount = PyList_GET_SIZE(revs);
 	static const Py_ssize_t capacity = 24;
 	int *depth, *interesting = NULL;
 	int i, j, v, ninteresting;
 	PyObject *dict = NULL, *keys = NULL;
 	long *seen = NULL;
 	int maxrev = -1;
 	long final;
 
 	if (revcount > capacity) {
 		PyErr_Format(PyExc_OverflowError,
 		             "bitset size (%ld) > capacity (%ld)",
 		             (long)revcount, (long)capacity);
 		return NULL;
 	}
 
 	for (i = 0; i < revcount; i++) {
 		int n = (int)PyInt_AsLong(PyList_GET_ITEM(revs, i));
 		if (n > maxrev)
 			maxrev = n;
 	}
 
 	depth = calloc(sizeof(*depth), maxrev + 1);
 	if (depth == NULL)
 		return PyErr_NoMemory();
 
 	seen = calloc(sizeof(*seen), maxrev + 1);
 	if (seen == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	interesting = calloc(sizeof(*interesting), ((size_t)1) << revcount);
 	if (interesting == NULL) {
 		PyErr_NoMemory();
 		goto bail;
 	}
 
 	if (PyList_Sort(revs) == -1)
 		goto bail;
 
 	for (i = 0; i < revcount; i++) {
 		int n = (int)PyInt_AsLong(PyList_GET_ITEM(revs, i));
 		long b = 1l << i;
 		depth[n] = 1;
 		seen[n] = b;
 		interesting[b] = 1;
 	}
 
 	/* invariant: ninteresting is the number of non-zero entries in
 	 * interesting. */
 	ninteresting = (int)revcount;
 
 	for (v = maxrev; v >= 0 && ninteresting > 1; v--) {
 		int dv = depth[v];
 		int parents[2];
 		long sv;
 
 		if (dv == 0)
 			continue;
 
 		sv = seen[v];
 		if (index_get_parents(self, v, parents, maxrev) < 0)
 			goto bail;
 
 		for (i = 0; i < 2; i++) {
 			int p = parents[i];
 			long sp;
 			int dp;
 
 			if (p == -1)
 				continue;
 
 			dp = depth[p];
 			sp = seen[p];
 			if (dp <= dv) {
 				depth[p] = dv + 1;
 				if (sp != sv) {
 					interesting[sv] += 1;
 					seen[p] = sv;
 					if (sp) {
 						interesting[sp] -= 1;
 						if (interesting[sp] == 0)
 							ninteresting -= 1;
 					}
 				}
 			} else if (dv == dp - 1) {
 				long nsp = sp | sv;
 				if (nsp == sp)
 					continue;
 				seen[p] = nsp;
 				interesting[sp] -= 1;
 				if (interesting[sp] == 0)
 					ninteresting -= 1;
 				if (interesting[nsp] == 0)
 					ninteresting += 1;
 				interesting[nsp] += 1;
 			}
 		}
 		interesting[sv] -= 1;
 		if (interesting[sv] == 0)
 			ninteresting -= 1;
 	}
 
 	final = 0;
 	j = ninteresting;
 	for (i = 0; i < (int)(2 << revcount) && j > 0; i++) {
 		if (interesting[i] == 0)
 			continue;
 		final |= i;
 		j -= 1;
 	}
 	if (final == 0) {
 		keys = PyList_New(0);
 		goto bail;
 	}
 
 	dict = PyDict_New();
 	if (dict == NULL)
 		goto bail;
 
 	for (i = 0; i < revcount; i++) {
 		PyObject *key;
 
 		if ((final & (1 << i)) == 0)
 			continue;
 
 		key = PyList_GET_ITEM(revs, i);
 		Py_INCREF(key);
 		Py_INCREF(Py_None);
 		if (PyDict_SetItem(dict, key, Py_None) == -1) {
 			Py_DECREF(key);
 			Py_DECREF(Py_None);
 			goto bail;
 		}
 	}
 
 	keys = PyDict_Keys(dict);
 
 bail:
 	free(depth);
 	free(seen);
 	free(interesting);
 	Py_XDECREF(dict);
 
 	return keys;
 }
 
 /*
  * Given a (possibly overlapping) set of revs, return all the
  * common ancestors heads: heads(::args[0] and ::a[1] and ...)
  */
 static PyObject *index_commonancestorsheads(indexObject *self, PyObject *args)
 {
 	PyObject *ret = NULL;
 	Py_ssize_t argcount, i, len;
 	bitmask repeat = 0;
 	int revcount = 0;
 	int *revs;
 
 	argcount = PySequence_Length(args);
 	revs = PyMem_Malloc(argcount * sizeof(*revs));
 	if (argcount > 0 && revs == NULL)
 		return PyErr_NoMemory();
 	len = index_length(self);
 
 	for (i = 0; i < argcount; i++) {
 		static const int capacity = 24;
 		PyObject *obj = PySequence_GetItem(args, i);
 		bitmask x;
 		long val;
 
 		if (!PyInt_Check(obj)) {
 			PyErr_SetString(PyExc_TypeError,
 			                "arguments must all be ints");
 			Py_DECREF(obj);
 			goto bail;
 		}
 		val = PyInt_AsLong(obj);
 		Py_DECREF(obj);
 		if (val == -1) {
 			ret = PyList_New(0);
 			goto done;
 		}
 		if (val < 0 || val >= len) {
 			PyErr_SetString(PyExc_IndexError, "index out of range");
 			goto bail;
 		}
 		/* this cheesy bloom filter lets us avoid some more
 		 * expensive duplicate checks in the common set-is-disjoint
 		 * case */
 		x = 1ull << (val & 0x3f);
 		if (repeat & x) {
 			int k;
 			for (k = 0; k < revcount; k++) {
 				if (val == revs[k])
 					goto duplicate;
 			}
 		} else
 			repeat |= x;
 		if (revcount >= capacity) {
 			PyErr_Format(PyExc_OverflowError,
 			             "bitset size (%d) > capacity (%d)",
 			             revcount, capacity);
 			goto bail;
 		}
 		revs[revcount++] = (int)val;
 	duplicate:;
 	}
 
 	if (revcount == 0) {
 		ret = PyList_New(0);
 		goto done;
 	}
 	if (revcount == 1) {
 		PyObject *obj;
 		ret = PyList_New(1);
 		if (ret == NULL)
 			goto bail;
 		obj = PyInt_FromLong(revs[0]);
 		if (obj == NULL)
 			goto bail;
 		PyList_SET_ITEM(ret, 0, obj);
 		goto done;
 	}
 
 	ret = find_gca_candidates(self, revs, revcount);
 	if (ret == NULL)
 		goto bail;
 
 done:
 	PyMem_Free(revs);
 	return ret;
 
 bail:
 	PyMem_Free(revs);
 	Py_XDECREF(ret);
 	return NULL;
 }
 
 /*
  * Given a (possibly overlapping) set of revs, return the greatest
  * common ancestors: those with the longest path to the root.
  */
 static PyObject *index_ancestors(indexObject *self, PyObject *args)
 {
 	PyObject *ret;
 	PyObject *gca = index_commonancestorsheads(self, args);
 	if (gca == NULL)
 		return NULL;
 
 	if (PyList_GET_SIZE(gca) <= 1) {
 		return gca;
 	}
 
 	ret = find_deepest(self, gca);
 	Py_DECREF(gca);
 	return ret;
 }
 
 /*
  * Invalidate any trie entries introduced by added revs.
  */
 static void index_invalidate_added(indexObject *self, Py_ssize_t start)
 {
 	Py_ssize_t i, len;
 
 	len = self->length + self->new_length;
 	i = start - self->length;
 	if (i < 0)
 		return;
 
 	for (i = start; i < len; i++)
 		nt_delete_node(&self->nt, index_deref(self, i) + 32);
 
 	self->new_length = start - self->length;
 }
 
 /*
  * Delete a numeric range of revs, which must be at the end of the
  * range.
  */
 static int index_slice_del(indexObject *self, PyObject *item)
 {
 	Py_ssize_t start, stop, step, slicelength;
 	Py_ssize_t length = index_length(self) + 1;
 	int ret = 0;
 
 /* Argument changed from PySliceObject* to PyObject* in Python 3. */
 #ifdef IS_PY3K
 	if (PySlice_GetIndicesEx(item, length, &start, &stop, &step,
 	                         &slicelength) < 0)
 #else
 	if (PySlice_GetIndicesEx((PySliceObject *)item, length, &start, &stop,
 	                         &step, &slicelength) < 0)
 #endif
 		return -1;
 
 	if (slicelength <= 0)
 		return 0;
 
 	if ((step < 0 && start < stop) || (step > 0 && start > stop))
 		stop = start;
 
 	if (step < 0) {
 		stop = start + 1;
 		start = stop + step * (slicelength - 1) - 1;
 		step = -step;
 	}
 
 	if (step != 1) {
 		PyErr_SetString(PyExc_ValueError,
 		                "revlog index delete requires step size of 1");
 		return -1;
 	}
 
 	if (stop != length - 1) {
 		PyErr_SetString(PyExc_IndexError,
 		                "revlog index deletion indices are invalid");
 		return -1;
 	}
 
 	if (start < self->length) {
 		if (self->ntinitialized) {
 			Py_ssize_t i;
 
 			for (i = start; i < self->length; i++) {
 				const char *node = index_node_existing(self, i);
 				if (node == NULL)
 					return -1;
 
 				nt_delete_node(&self->nt, node);
 			}
 			if (self->new_length)
 				index_invalidate_added(self, self->length);
 			if (self->ntrev > start)
 				self->ntrev = (int)start;
 		} else if (self->new_length) {
 			self->new_length = 0;
 		}
 
 		self->length = start;
 		goto done;
 	}
 
 	if (self->ntinitialized) {
 		index_invalidate_added(self, start);
 		if (self->ntrev > start)
 			self->ntrev = (int)start;
 	} else {
 		self->new_length = start - self->length;
 	}
 done:
 	Py_CLEAR(self->headrevs);
 	return ret;
 }
 
 /*
  * Supported ops:
  *
  * slice deletion
  * string assignment (extend node->rev mapping)
  * string deletion (shrink node->rev mapping)
  */
 static int index_assign_subscript(indexObject *self, PyObject *item,
                                   PyObject *value)
 {
 	char *node;
 	long rev;
 
 	if (PySlice_Check(item) && value == NULL)
 		return index_slice_del(self, item);
 
 	if (node_check(self->nodelen, item, &node) == -1)
 		return -1;
 
 	if (value == NULL)
 		return self->ntinitialized ? nt_delete_node(&self->nt, node)
 		                           : 0;
 	rev = PyInt_AsLong(value);
 	if (rev > INT_MAX || rev < 0) {
 		if (!PyErr_Occurred())
 			PyErr_SetString(PyExc_ValueError, "rev out of range");
 		return -1;
 	}
 
 	if (index_init_nt(self) == -1)
 		return -1;
 	return nt_insert(&self->nt, node, (int)rev);
 }
 
 /*
  * Find all RevlogNG entries in an index that has inline data. Update
  * the optional "offsets" table with those entries.
  */
 static Py_ssize_t inline_scan(indexObject *self, const char **offsets)
 {
 	const char *data = (const char *)self->buf.buf;
 	Py_ssize_t pos = 0;
 	Py_ssize_t end = self->buf.len;
 	long incr = self->entry_size;
 	Py_ssize_t len = 0;
 
 	while (pos + self->entry_size <= end && pos >= 0) {
 		uint32_t comp_len, sidedata_comp_len = 0;
 		/* 3rd element of header is length of compressed inline data */
 		comp_len = getbe32(data + pos + 8);
 		if (self->entry_size == v2_entry_size) {
 			sidedata_comp_len = getbe32(data + pos + 72);
 		}
 		incr = self->entry_size + comp_len + sidedata_comp_len;
 		if (offsets)
 			offsets[len] = data + pos;
 		len++;
 		pos += incr;
 	}
 
 	if (pos != end) {
 		if (!PyErr_Occurred())
 			PyErr_SetString(PyExc_ValueError, "corrupt index file");
 		return -1;
 	}
 
 	return len;
 }
 
 static int index_init(indexObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *data_obj, *inlined_obj, *revlogv2;
 	Py_ssize_t size;
 
 	static char *kwlist[] = {"data", "inlined", "revlogv2", NULL};
 
 	/* Initialize before argument-checking to avoid index_dealloc() crash.
 	 */
 	self->added = NULL;
 	self->new_length = 0;
 	self->added_length = 0;
 	self->data = NULL;
 	memset(&self->buf, 0, sizeof(self->buf));
 	self->headrevs = NULL;
 	self->filteredrevs = Py_None;
 	Py_INCREF(Py_None);
 	self->ntinitialized = 0;
 	self->offsets = NULL;
 	self->nodelen = 20;
 	self->nullentry = NULL;
+	self->rust_ext_compat = 1;
 
 	revlogv2 = NULL;
 	if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|O", kwlist,
 	                                 &data_obj, &inlined_obj, &revlogv2))
 		return -1;
 	if (!PyObject_CheckBuffer(data_obj)) {
 		PyErr_SetString(PyExc_TypeError,
 		                "data does not support buffer interface");
 		return -1;
 	}
 	if (self->nodelen < 20 || self->nodelen > (Py_ssize_t)sizeof(nullid)) {
 		PyErr_SetString(PyExc_RuntimeError, "unsupported node size");
 		return -1;
 	}
 
 	if (revlogv2 && PyObject_IsTrue(revlogv2)) {
 		self->format_version = format_v2;
 		self->entry_size = v2_entry_size;
 	} else {
 		self->format_version = format_v1;
 		self->entry_size = v1_entry_size;
 	}
 
 	self->nullentry = Py_BuildValue(
 	    PY23("iiiiiiis#iiBB", "iiiiiiiy#iiBB"), 0, 0, 0, -1, -1, -1, -1,
 	    nullid, self->nodelen, 0, 0, comp_mode_inline, comp_mode_inline);
 
 	if (!self->nullentry)
 		return -1;
 	PyObject_GC_UnTrack(self->nullentry);
 
 	if (PyObject_GetBuffer(data_obj, &self->buf, PyBUF_SIMPLE) == -1)
 		return -1;
 	size = self->buf.len;
 
 	self->inlined = inlined_obj && PyObject_IsTrue(inlined_obj);
 	self->data = data_obj;
 
 	self->ntlookups = self->ntmisses = 0;
 	self->ntrev = -1;
 	Py_INCREF(self->data);
 
 	if (self->inlined) {
 		Py_ssize_t len = inline_scan(self, NULL);
 		if (len == -1)
 			goto bail;
 		self->length = len;
 	} else {
 		if (size % self->entry_size) {
 			PyErr_SetString(PyExc_ValueError, "corrupt index file");
 			goto bail;
 		}
 		self->length = size / self->entry_size;
 	}
 
 	return 0;
 bail:
 	return -1;
 }
 
 static PyObject *index_nodemap(indexObject *self)
 {
 	Py_INCREF(self);
 	return (PyObject *)self;
 }
 
 static void _index_clearcaches(indexObject *self)
 {
 	if (self->offsets) {
 		PyMem_Free((void *)self->offsets);
 		self->offsets = NULL;
 	}
 	if (self->ntinitialized) {
 		nt_dealloc(&self->nt);
 	}
 	self->ntinitialized = 0;
 	Py_CLEAR(self->headrevs);
 }
 
 static PyObject *index_clearcaches(indexObject *self)
 {
 	_index_clearcaches(self);
 	self->ntrev = -1;
 	self->ntlookups = self->ntmisses = 0;
 	Py_RETURN_NONE;
 }
 
 static void index_dealloc(indexObject *self)
 {
 	_index_clearcaches(self);
 	Py_XDECREF(self->filteredrevs);
 	if (self->buf.buf) {
 		PyBuffer_Release(&self->buf);
 		memset(&self->buf, 0, sizeof(self->buf));
 	}
 	Py_XDECREF(self->data);
 	PyMem_Free(self->added);
 	Py_XDECREF(self->nullentry);
 	PyObject_Del(self);
 }
 
 static PySequenceMethods index_sequence_methods = {
     (lenfunc)index_length,      /* sq_length */
     0,                          /* sq_concat */
     0,                          /* sq_repeat */
     (ssizeargfunc)index_get,    /* sq_item */
     0,                          /* sq_slice */
     0,                          /* sq_ass_item */
     0,                          /* sq_ass_slice */
     (objobjproc)index_contains, /* sq_contains */
 };
 
 static PyMappingMethods index_mapping_methods = {
     (lenfunc)index_length,                 /* mp_length */
     (binaryfunc)index_getitem,             /* mp_subscript */
     (objobjargproc)index_assign_subscript, /* mp_ass_subscript */
 };
 
 static PyMethodDef index_methods[] = {
     {"ancestors", (PyCFunction)index_ancestors, METH_VARARGS,
      "return the gca set of the given revs"},
     {"commonancestorsheads", (PyCFunction)index_commonancestorsheads,
      METH_VARARGS,
      "return the heads of the common ancestors of the given revs"},
     {"clearcaches", (PyCFunction)index_clearcaches, METH_NOARGS,
      "clear the index caches"},
     {"get", (PyCFunction)index_m_get, METH_VARARGS, "get an index entry"},
     {"get_rev", (PyCFunction)index_m_get, METH_VARARGS,
      "return `rev` associated with a node or None"},
     {"has_node", (PyCFunction)index_m_has_node, METH_O,
      "return True if the node exist in the index"},
     {"rev", (PyCFunction)index_m_rev, METH_O,
      "return `rev` associated with a node or raise RevlogError"},
     {"computephasesmapsets", (PyCFunction)compute_phases_map_sets, METH_VARARGS,
      "compute phases"},
     {"reachableroots2", (PyCFunction)reachableroots2, METH_VARARGS,
      "reachableroots"},
     {"replace_sidedata_info", (PyCFunction)index_replace_sidedata_info,
      METH_VARARGS, "replace an existing index entry with a new value"},
     {"headrevs", (PyCFunction)index_headrevs, METH_VARARGS,
      "get head revisions"}, /* Can do filtering since 3.2 */
     {"headrevsfiltered", (PyCFunction)index_headrevs, METH_VARARGS,
      "get filtered head revisions"}, /* Can always do filtering */
     {"issnapshot", (PyCFunction)index_issnapshot, METH_O,
      "True if the object is a snapshot"},
     {"findsnapshots", (PyCFunction)index_findsnapshots, METH_VARARGS,
      "Gather snapshot data in a cache dict"},
     {"deltachain", (PyCFunction)index_deltachain, METH_VARARGS,
      "determine revisions with deltas to reconstruct fulltext"},
     {"slicechunktodensity", (PyCFunction)index_slicechunktodensity,
      METH_VARARGS, "determine revisions with deltas to reconstruct fulltext"},
     {"append", (PyCFunction)index_append, METH_O, "append an index entry"},
     {"partialmatch", (PyCFunction)index_partialmatch, METH_VARARGS,
      "match a potentially ambiguous node ID"},
     {"shortest", (PyCFunction)index_shortest, METH_VARARGS,
      "find length of shortest hex nodeid of a binary ID"},
     {"stats", (PyCFunction)index_stats, METH_NOARGS, "stats for the index"},
     {"entry_binary", (PyCFunction)index_entry_binary, METH_O,
      "return an entry in binary form"},
     {"pack_header", (PyCFunction)index_pack_header, METH_VARARGS,
      "pack the revlog header information into binary"},
     {NULL} /* Sentinel */
 };
 
 static PyGetSetDef index_getset[] = {
     {"nodemap", (getter)index_nodemap, NULL, "nodemap", NULL},
     {NULL} /* Sentinel */
 };
 
 static PyMemberDef index_members[] = {
     {"entry_size", T_LONG, offsetof(indexObject, entry_size), 0,
      "size of an index entry"},
+    {"rust_ext_compat", T_LONG, offsetof(indexObject, rust_ext_compat), 0,
+     "size of an index entry"},
     {NULL} /* Sentinel */
 };
 
 PyTypeObject HgRevlogIndex_Type = {
     PyVarObject_HEAD_INIT(NULL, 0) /* header */
     "parsers.index",               /* tp_name */
     sizeof(indexObject),           /* tp_basicsize */
     0,                             /* tp_itemsize */
     (destructor)index_dealloc,     /* tp_dealloc */
     0,                             /* tp_print */
     0,                             /* tp_getattr */
     0,                             /* tp_setattr */
     0,                             /* tp_compare */
     0,                             /* tp_repr */
     0,                             /* tp_as_number */
     &index_sequence_methods,       /* tp_as_sequence */
     &index_mapping_methods,        /* tp_as_mapping */
     0,                             /* tp_hash */
     0,                             /* tp_call */
     0,                             /* tp_str */
     0,                             /* tp_getattro */
     0,                             /* tp_setattro */
     0,                             /* tp_as_buffer */
     Py_TPFLAGS_DEFAULT,            /* tp_flags */
     "revlog index",                /* tp_doc */
     0,                             /* tp_traverse */
     0,                             /* tp_clear */
     0,                             /* tp_richcompare */
     0,                             /* tp_weaklistoffset */
     0,                             /* tp_iter */
     0,                             /* tp_iternext */
     index_methods,                 /* tp_methods */
     index_members,                 /* tp_members */
     index_getset,                  /* tp_getset */
     0,                             /* tp_base */
     0,                             /* tp_dict */
     0,                             /* tp_descr_get */
     0,                             /* tp_descr_set */
     0,                             /* tp_dictoffset */
     (initproc)index_init,          /* tp_init */
     0,                             /* tp_alloc */
 };
 
 /*
  * returns a tuple of the form (index, cache) with elements as
  * follows:
  *
  * index: an index object that lazily parses Revlog (v1 or v2) records
  * cache: if data is inlined, a tuple (0, index_file_content), else None
  *        index_file_content could be a string, or a buffer
  *
  * added complications are for backwards compatibility
  */
 PyObject *parse_index2(PyObject *self, PyObject *args, PyObject *kwargs)
 {
 	PyObject *cache = NULL;
 	indexObject *idx;
 	int ret;
 
 	idx = PyObject_New(indexObject, &HgRevlogIndex_Type);
 	if (idx == NULL)
 		goto bail;
 
 	ret = index_init(idx, args, kwargs);
 	if (ret == -1)
 		goto bail;
 
 	if (idx->inlined) {
 		cache = Py_BuildValue("iO", 0, idx->data);
 		if (cache == NULL)
 			goto bail;
 	} else {
 		cache = Py_None;
 		Py_INCREF(cache);
 	}
 
 	return Py_BuildValue("NN", idx, cache);
 
 bail:
 	Py_XDECREF(idx);
 	Py_XDECREF(cache);
 	return NULL;
 }
 
 static Revlog_CAPI CAPI = {
     /* increment the abi_version field upon each change in the Revlog_CAPI
        struct or in the ABI of the listed functions */
     2,
     index_length,
     index_node,
     HgRevlogIndex_GetParents,
 };
 
 void revlog_module_init(PyObject *mod)
 {
 	PyObject *caps = NULL;
 	HgRevlogIndex_Type.tp_new = PyType_GenericNew;
 	if (PyType_Ready(&HgRevlogIndex_Type) < 0)
 		return;
 	Py_INCREF(&HgRevlogIndex_Type);
 	PyModule_AddObject(mod, "index", (PyObject *)&HgRevlogIndex_Type);
 
 	nodetreeType.tp_new = PyType_GenericNew;
 	if (PyType_Ready(&nodetreeType) < 0)
 		return;
 	Py_INCREF(&nodetreeType);
 	PyModule_AddObject(mod, "nodetree", (PyObject *)&nodetreeType);
 
 	caps = PyCapsule_New(&CAPI, "mercurial.cext.parsers.revlog_CAPI", NULL);
 	if (caps != NULL)
 		PyModule_AddObject(mod, "revlog_CAPI", caps);
 }

mercurial/policy.py

@@ -1,157 +1,157
 # policy.py - module policy logic for Mercurial.
 #
 # Copyright 2015 Gregory Szorc <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import os
 import sys
 
 from .pycompat import getattr
 
 # Rules for how modules can be loaded. Values are:
 #
 #    c - require C extensions
 #    rust+c - require Rust and C extensions
 #    rust+c-allow - allow Rust and C extensions with fallback to pure Python
 #                   for each
 #    allow - allow pure Python implementation when C loading fails
 #    cffi - required cffi versions (implemented within pure module)
 #    cffi-allow - allow pure Python implementation if cffi version is missing
 #    py - only load pure Python modules
 #
 # By default, fall back to the pure modules so the in-place build can
 # run without recompiling the C extensions. This will be overridden by
 # __modulepolicy__ generated by setup.py.
 policy = b'allow'
 _packageprefs = {
     # policy: (versioned package, pure package)
     b'c': ('cext', None),
     b'allow': ('cext', 'pure'),
     b'cffi': ('cffi', None),
     b'cffi-allow': ('cffi', 'pure'),
     b'py': (None, 'pure'),
     # For now, rust policies impact importrust only
     b'rust+c': ('cext', None),
     b'rust+c-allow': ('cext', 'pure'),
 }
 
 try:
     from . import __modulepolicy__
 
     policy = __modulepolicy__.modulepolicy
 except ImportError:
     pass
 
 # PyPy doesn't load C extensions.
 #
 # The canonical way to do this is to test platform.python_implementation().
 # But we don't import platform and don't bloat for it here.
 if '__pypy__' in sys.builtin_module_names:
     policy = b'cffi'
 
 # Environment variable can always force settings.
 if sys.version_info[0] >= 3:
     if 'HGMODULEPOLICY' in os.environ:
         policy = os.environ['HGMODULEPOLICY'].encode('utf-8')
 else:
     policy = os.environ.get('HGMODULEPOLICY', policy)
 
 
 def _importfrom(pkgname, modname):
     # from .<pkgname> import <modname> (where . is looked through this module)
     fakelocals = {}
     pkg = __import__(pkgname, globals(), fakelocals, [modname], level=1)
     try:
         fakelocals[modname] = mod = getattr(pkg, modname)
     except AttributeError:
         raise ImportError('cannot import name %s' % modname)
     # force import; fakelocals[modname] may be replaced with the real module
     getattr(mod, '__doc__', None)
     return fakelocals[modname]
 
 
 # keep in sync with "version" in C modules
 _cextversions = {
     ('cext', 'base85'): 1,
     ('cext', 'bdiff'): 3,
     ('cext', 'mpatch'): 1,
     ('cext', 'osutil'): 4,
-    ('cext', 'parsers'): 19,
+    ('cext', 'parsers'): 20,
 }
 
 # map import request to other package or module
 _modredirects = {
     ('cext', 'charencode'): ('cext', 'parsers'),
     ('cffi', 'base85'): ('pure', 'base85'),
     ('cffi', 'charencode'): ('pure', 'charencode'),
     ('cffi', 'parsers'): ('pure', 'parsers'),
 }
 
 
 def _checkmod(pkgname, modname, mod):
     expected = _cextversions.get((pkgname, modname))
     actual = getattr(mod, 'version', None)
     if actual != expected:
         raise ImportError(
             'cannot import module %s.%s '
             '(expected version: %d, actual: %r)'
             % (pkgname, modname, expected, actual)
         )
 
 
 def importmod(modname):
     """Import module according to policy and check API version"""
     try:
         verpkg, purepkg = _packageprefs[policy]
     except KeyError:
         raise ImportError('invalid HGMODULEPOLICY %r' % policy)
     assert verpkg or purepkg
     if verpkg:
         pn, mn = _modredirects.get((verpkg, modname), (verpkg, modname))
         try:
             mod = _importfrom(pn, mn)
             if pn == verpkg:
                 _checkmod(pn, mn, mod)
             return mod
         except ImportError:
             if not purepkg:
                 raise
     pn, mn = _modredirects.get((purepkg, modname), (purepkg, modname))
     return _importfrom(pn, mn)
 
 
 def _isrustpermissive():
     """Assuming the policy is a Rust one, tell if it's permissive."""
     return policy.endswith(b'-allow')
 
 
 def importrust(modname, member=None, default=None):
     """Import Rust module according to policy and availability.
 
     If policy isn't a Rust one, this returns `default`.
 
     If either the module or its member is not available, this returns `default`
     if policy is permissive and raises `ImportError` if not.
     """
     if not policy.startswith(b'rust'):
         return default
 
     try:
         mod = _importfrom('rustext', modname)
     except ImportError:
         if _isrustpermissive():
             return default
         raise
     if member is None:
         return mod
 
     try:
         return getattr(mod, member)
     except AttributeError:
         if _isrustpermissive():
             return default
         raise ImportError("Cannot import name %s" % member)

mercurial/pure/parsers.py

@@ -1,406 +1,408
 # parsers.py - Python implementation of parsers.c
 #
 # Copyright 2009 Olivia Mackall <[e-mail]> and others
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
 from __future__ import absolute_import
 
 import struct
 import zlib
 
 from ..node import (
     nullrev,
     sha1nodeconstants,
 )
 from .. import (
     error,
     pycompat,
     util,
 )
 
 from ..revlogutils import nodemap as nodemaputil
 from ..revlogutils import constants as revlog_constants
 
 stringio = pycompat.bytesio
 
 
 _pack = struct.pack
 _unpack = struct.unpack
 _compress = zlib.compress
 _decompress = zlib.decompress
 
 # Some code below makes tuples directly because it's more convenient. However,
 # code outside this module should always use dirstatetuple.
 def dirstatetuple(*x):
     # x is a tuple
     return x
 
 
 def gettype(q):
     return int(q & 0xFFFF)
 
 
 def offset_type(offset, type):
     return int(int(offset) << 16 | type)
 
 
 class BaseIndexObject(object):
+    # Can I be passed to an algorithme implemented in Rust ?
+    rust_ext_compat = 0
     # Format of an index entry according to Python's `struct` language
     index_format = revlog_constants.INDEX_ENTRY_V1
     # Size of a C unsigned long long int, platform independent
     big_int_size = struct.calcsize(b'>Q')
     # Size of a C long int, platform independent
     int_size = struct.calcsize(b'>i')
     # An empty index entry, used as a default value to be overridden, or nullrev
     null_item = (
         0,
         0,
         0,
         -1,
         -1,
         -1,
         -1,
         sha1nodeconstants.nullid,
         0,
         0,
         revlog_constants.COMP_MODE_INLINE,
         revlog_constants.COMP_MODE_INLINE,
     )
 
     @util.propertycache
     def entry_size(self):
         return self.index_format.size
 
     @property
     def nodemap(self):
         msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
         util.nouideprecwarn(msg, b'5.3', stacklevel=2)
         return self._nodemap
 
     @util.propertycache
     def _nodemap(self):
         nodemap = nodemaputil.NodeMap({sha1nodeconstants.nullid: nullrev})
         for r in range(0, len(self)):
             n = self[r][7]
             nodemap[n] = r
         return nodemap
 
     def has_node(self, node):
         """return True if the node exist in the index"""
         return node in self._nodemap
 
     def rev(self, node):
         """return a revision for a node
 
         If the node is unknown, raise a RevlogError"""
         return self._nodemap[node]
 
     def get_rev(self, node):
         """return a revision for a node
 
         If the node is unknown, return None"""
         return self._nodemap.get(node)
 
     def _stripnodes(self, start):
         if '_nodemap' in vars(self):
             for r in range(start, len(self)):
                 n = self[r][7]
                 del self._nodemap[n]
 
     def clearcaches(self):
         self.__dict__.pop('_nodemap', None)
 
     def __len__(self):
         return self._lgt + len(self._extra)
 
     def append(self, tup):
         if '_nodemap' in vars(self):
             self._nodemap[tup[7]] = len(self)
         data = self._pack_entry(len(self), tup)
         self._extra.append(data)
 
     def _pack_entry(self, rev, entry):
         assert entry[8] == 0
         assert entry[9] == 0
         return self.index_format.pack(*entry[:8])
 
     def _check_index(self, i):
         if not isinstance(i, int):
             raise TypeError(b"expecting int indexes")
         if i < 0 or i >= len(self):
             raise IndexError
 
     def __getitem__(self, i):
         if i == -1:
             return self.null_item
         self._check_index(i)
         if i >= self._lgt:
             data = self._extra[i - self._lgt]
         else:
             index = self._calculate_index(i)
             data = self._data[index : index + self.entry_size]
         r = self._unpack_entry(i, data)
         if self._lgt and i == 0:
             r = (offset_type(0, gettype(r[0])),) + r[1:]
         return r
 
     def _unpack_entry(self, rev, data):
         r = self.index_format.unpack(data)
         r = r + (
             0,
             0,
             revlog_constants.COMP_MODE_INLINE,
             revlog_constants.COMP_MODE_INLINE,
         )
         return r
 
     def pack_header(self, header):
         """pack header information as binary"""
         v_fmt = revlog_constants.INDEX_HEADER
         return v_fmt.pack(header)
 
     def entry_binary(self, rev):
         """return the raw binary string representing a revision"""
         entry = self[rev]
         p = revlog_constants.INDEX_ENTRY_V1.pack(*entry[:8])
         if rev == 0:
             p = p[revlog_constants.INDEX_HEADER.size :]
         return p
 
 
 class IndexObject(BaseIndexObject):
     def __init__(self, data):
         assert len(data) % self.entry_size == 0, (
             len(data),
             self.entry_size,
             len(data) % self.entry_size,
         )
         self._data = data
         self._lgt = len(data) // self.entry_size
         self._extra = []
 
     def _calculate_index(self, i):
         return i * self.entry_size
 
     def __delitem__(self, i):
         if not isinstance(i, slice) or not i.stop == -1 or i.step is not None:
             raise ValueError(b"deleting slices only supports a:-1 with step 1")
         i = i.start
         self._check_index(i)
         self._stripnodes(i)
         if i < self._lgt:
             self._data = self._data[: i * self.entry_size]
             self._lgt = i
             self._extra = []
         else:
             self._extra = self._extra[: i - self._lgt]
 
 
 class PersistentNodeMapIndexObject(IndexObject):
     """a Debug oriented class to test persistent nodemap
 
     We need a simple python object to test API and higher level behavior. See
     the Rust implementation for  more serious usage. This should be used only
     through the dedicated `devel.persistent-nodemap` config.
     """
 
     def nodemap_data_all(self):
         """Return bytes containing a full serialization of a nodemap
 
         The nodemap should be valid for the full set of revisions in the
         index."""
         return nodemaputil.persistent_data(self)
 
     def nodemap_data_incremental(self):
         """Return bytes containing a incremental update to persistent nodemap
 
         This containst the data for an append-only update of the data provided
         in the last call to `update_nodemap_data`.
         """
         if self._nm_root is None:
             return None
         docket = self._nm_docket
         changed, data = nodemaputil.update_persistent_data(
             self, self._nm_root, self._nm_max_idx, self._nm_docket.tip_rev
         )
 
         self._nm_root = self._nm_max_idx = self._nm_docket = None
         return docket, changed, data
 
     def update_nodemap_data(self, docket, nm_data):
         """provide full block of persisted binary data for a nodemap
 
         The data are expected to come from disk. See `nodemap_data_all` for a
         produceur of such data."""
         if nm_data is not None:
             self._nm_root, self._nm_max_idx = nodemaputil.parse_data(nm_data)
             if self._nm_root:
                 self._nm_docket = docket
             else:
                 self._nm_root = self._nm_max_idx = self._nm_docket = None
 
 
 class InlinedIndexObject(BaseIndexObject):
     def __init__(self, data, inline=0):
         self._data = data
         self._lgt = self._inline_scan(None)
         self._inline_scan(self._lgt)
         self._extra = []
 
     def _inline_scan(self, lgt):
         off = 0
         if lgt is not None:
             self._offsets = [0] * lgt
         count = 0
         while off <= len(self._data) - self.entry_size:
             start = off + self.big_int_size
             (s,) = struct.unpack(
                 b'>i',
                 self._data[start : start + self.int_size],
             )
             if lgt is not None:
                 self._offsets[count] = off
             count += 1
             off += self.entry_size + s
         if off != len(self._data):
             raise ValueError(b"corrupted data")
         return count
 
     def __delitem__(self, i):
         if not isinstance(i, slice) or not i.stop == -1 or i.step is not None:
             raise ValueError(b"deleting slices only supports a:-1 with step 1")
         i = i.start
         self._check_index(i)
         self._stripnodes(i)
         if i < self._lgt:
             self._offsets = self._offsets[:i]
             self._lgt = i
             self._extra = []
         else:
             self._extra = self._extra[: i - self._lgt]
 
     def _calculate_index(self, i):
         return self._offsets[i]
 
 
 def parse_index2(data, inline, revlogv2=False):
     if not inline:
         cls = IndexObject2 if revlogv2 else IndexObject
         return cls(data), None
     cls = InlinedIndexObject
     return cls(data, inline), (0, data)
 
 
 class IndexObject2(IndexObject):
     index_format = revlog_constants.INDEX_ENTRY_V2
 
     def replace_sidedata_info(
         self,
         rev,
         sidedata_offset,
         sidedata_length,
         offset_flags,
         compression_mode,
     ):
         """
         Replace an existing index entry's sidedata offset and length with new
         ones.
         This cannot be used outside of the context of sidedata rewriting,
         inside the transaction that creates the revision `rev`.
         """
         if rev < 0:
             raise KeyError
         self._check_index(rev)
         if rev < self._lgt:
             msg = b"cannot rewrite entries outside of this transaction"
             raise KeyError(msg)
         else:
             entry = list(self[rev])
             entry[0] = offset_flags
             entry[8] = sidedata_offset
             entry[9] = sidedata_length
             entry[11] = compression_mode
             entry = tuple(entry)
             new = self._pack_entry(rev, entry)
             self._extra[rev - self._lgt] = new
 
     def _unpack_entry(self, rev, data):
         data = self.index_format.unpack(data)
         entry = data[:10]
         data_comp = data[10] & 3
         sidedata_comp = (data[10] & (3 << 2)) >> 2
         return entry + (data_comp, sidedata_comp)
 
     def _pack_entry(self, rev, entry):
         data = entry[:10]
         data_comp = entry[10] & 3
         sidedata_comp = (entry[11] & 3) << 2
         data += (data_comp | sidedata_comp,)
 
         return self.index_format.pack(*data)
 
     def entry_binary(self, rev):
         """return the raw binary string representing a revision"""
         entry = self[rev]
         return self._pack_entry(rev, entry)
 
     def pack_header(self, header):
         """pack header information as binary"""
         msg = 'version header should go in the docket, not the index: %d'
         msg %= header
         raise error.ProgrammingError(msg)
 
 
 def parse_index_devel_nodemap(data, inline):
     """like parse_index2, but alway return a PersistentNodeMapIndexObject"""
     return PersistentNodeMapIndexObject(data), None
 
 
 def parse_dirstate(dmap, copymap, st):
     parents = [st[:20], st[20:40]]
     # dereference fields so they will be local in loop
     format = b">cllll"
     e_size = struct.calcsize(format)
     pos1 = 40
     l = len(st)
 
     # the inner loop
     while pos1 < l:
         pos2 = pos1 + e_size
         e = _unpack(b">cllll", st[pos1:pos2])  # a literal here is faster
         pos1 = pos2 + e[4]
         f = st[pos2:pos1]
         if b'\0' in f:
             f, c = f.split(b'\0')
             copymap[f] = c
         dmap[f] = e[:4]
     return parents
 
 
 def pack_dirstate(dmap, copymap, pl, now):
     now = int(now)
     cs = stringio()
     write = cs.write
     write(b"".join(pl))
     for f, e in pycompat.iteritems(dmap):
         if e[0] == b'n' and e[3] == now:
             # The file was last modified "simultaneously" with the current
             # write to dirstate (i.e. within the same second for file-
             # systems with a granularity of 1 sec). This commonly happens
             # for at least a couple of files on 'update'.
             # The user could change the file without changing its size
             # within the same second. Invalidate the file's mtime in
             # dirstate, forcing future 'status' calls to compare the
             # contents of the file if the size is the same. This prevents
             # mistakenly treating such files as clean.
             e = dirstatetuple(e[0], e[1], e[2], -1)
             dmap[f] = e
 
         if f in copymap:
             f = b"%s\0%s" % (f, copymap[f])
         e = _pack(b">cllll", e[0], e[1], e[2], e[3], len(f))
         write(e)
         write(f)
     return cs.getvalue()

mercurial/revlogutils/revlogv0.py

@@ -1,162 +1,163
 # revlogv0 - code related to revlog format "V0"
 #
 # Copyright 2005-2007 Olivia Mackall <[e-mail]>
 #
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 from __future__ import absolute_import
 
 
 from ..node import sha1nodeconstants
 from .constants import (
     COMP_MODE_INLINE,
     INDEX_ENTRY_V0,
 )
 from ..i18n import _
 
 from .. import (
     error,
     node,
     pycompat,
     util,
 )
 
 from . import (
     flagutil,
     nodemap as nodemaputil,
 )
 
 
 def getoffset(q):
     return int(q >> 16)
 
 
 def gettype(q):
     return int(q & 0xFFFF)
 
 
 def offset_type(offset, type):
     if (type & ~flagutil.REVIDX_KNOWN_FLAGS) != 0:
         raise ValueError(b'unknown revlog index flags')
     return int(int(offset) << 16 | type)
 
 
 class revlogoldindex(list):
+    rust_ext_compat = 0
     entry_size = INDEX_ENTRY_V0.size
     null_item = (
         0,
         0,
         0,
         -1,
         -1,
         -1,
         -1,
         sha1nodeconstants.nullid,
         0,
         0,
         COMP_MODE_INLINE,
         COMP_MODE_INLINE,
     )
 
     @property
     def nodemap(self):
         msg = b"index.nodemap is deprecated, use index.[has_node|rev|get_rev]"
         util.nouideprecwarn(msg, b'5.3', stacklevel=2)
         return self._nodemap
 
     @util.propertycache
     def _nodemap(self):
         nodemap = nodemaputil.NodeMap({sha1nodeconstants.nullid: node.nullrev})
         for r in range(0, len(self)):
             n = self[r][7]
             nodemap[n] = r
         return nodemap
 
     def has_node(self, node):
         """return True if the node exist in the index"""
         return node in self._nodemap
 
     def rev(self, node):
         """return a revision for a node
 
         If the node is unknown, raise a RevlogError"""
         return self._nodemap[node]
 
     def get_rev(self, node):
         """return a revision for a node
 
         If the node is unknown, return None"""
         return self._nodemap.get(node)
 
     def append(self, tup):
         self._nodemap[tup[7]] = len(self)
         super(revlogoldindex, self).append(tup)
 
     def __delitem__(self, i):
         if not isinstance(i, slice) or not i.stop == -1 or i.step is not None:
             raise ValueError(b"deleting slices only supports a:-1 with step 1")
         for r in pycompat.xrange(i.start, len(self)):
             del self._nodemap[self[r][7]]
         super(revlogoldindex, self).__delitem__(i)
 
     def clearcaches(self):
         self.__dict__.pop('_nodemap', None)
 
     def __getitem__(self, i):
         if i == -1:
             return self.null_item
         return list.__getitem__(self, i)
 
     def pack_header(self, header):
         """pack header information in binary"""
         return b''
 
     def entry_binary(self, rev):
         """return the raw binary string representing a revision"""
         entry = self[rev]
         if gettype(entry[0]):
             raise error.RevlogError(
                 _(b'index entry flags need revlog version 1')
             )
         e2 = (
             getoffset(entry[0]),
             entry[1],
             entry[3],
             entry[4],
             self[entry[5]][7],
             self[entry[6]][7],
             entry[7],
         )
         return INDEX_ENTRY_V0.pack(*e2)
 
 
 def parse_index_v0(data, inline):
     s = INDEX_ENTRY_V0.size
     index = []
     nodemap = nodemaputil.NodeMap({node.nullid: node.nullrev})
     n = off = 0
     l = len(data)
     while off + s <= l:
         cur = data[off : off + s]
         off += s
         e = INDEX_ENTRY_V0.unpack(cur)
         # transform to revlogv1 format
         e2 = (
             offset_type(e[0], 0),
             e[1],
             -1,
             e[2],
             e[3],
             nodemap.get(e[4], node.nullrev),
             nodemap.get(e[5], node.nullrev),
             e[6],
             0,  # no side data support
             0,  # no side data support
             COMP_MODE_INLINE,
         )
         index.append(e2)
         nodemap[e[6]] = n
         n += 1
 
     index = revlogoldindex(index)
     return index, None

rust/hg-cpython/src/cindex.rs

@@ -1,176 +1,183
 // cindex.rs
 //
 // Copyright 2018 Georges Racinet <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 //! Bindings to use the Index defined by the parsers C extension
 //!
 //! Ideally, we should use an Index entirely implemented in Rust,
 //! but this will take some time to get there.
 
 use cpython::{
-    exc::ImportError, ObjectProtocol, PyClone, PyErr, PyObject, PyResult,
-    PyTuple, Python, PythonObject,
+    exc::ImportError, exc::TypeError, ObjectProtocol, PyClone, PyErr,
+    PyObject, PyResult, PyTuple, Python, PythonObject,
 };
 use hg::revlog::{Node, RevlogIndex};
 use hg::{Graph, GraphError, Revision, WORKING_DIRECTORY_REVISION};
 use libc::{c_int, ssize_t};
 
 const REVLOG_CABI_VERSION: c_int = 2;
 
 #[repr(C)]
 pub struct Revlog_CAPI {
     abi_version: c_int,
     index_length:
         unsafe extern "C" fn(index: *mut revlog_capi::RawPyObject) -> ssize_t,
     index_node: unsafe extern "C" fn(
         index: *mut revlog_capi::RawPyObject,
         rev: ssize_t,
     ) -> *const Node,
     index_parents: unsafe extern "C" fn(
         index: *mut revlog_capi::RawPyObject,
         rev: c_int,
         ps: *mut [c_int; 2],
     ) -> c_int,
 }
 
 py_capsule!(
     from mercurial.cext.parsers import revlog_CAPI
         as revlog_capi for Revlog_CAPI);
 
 /// A `Graph` backed up by objects and functions from revlog.c
 ///
 /// This implementation of the `Graph` trait, relies on (pointers to)
 /// - the C index object (`index` member)
 /// - the `index_get_parents()` function (`parents` member)
 ///
 /// # Safety
 ///
 /// The C index itself is mutable, and this Rust exposition is **not
 /// protected by the GIL**, meaning that this construct isn't safe with respect
 /// to Python threads.
 ///
 /// All callers of this `Index` must acquire the GIL and must not release it
 /// while working.
 ///
 /// # TODO find a solution to make it GIL safe again.
 ///
 /// This is non trivial, and can wait until we have a clearer picture with
 /// more Rust Mercurial constructs.
 ///
 /// One possibility would be to a `GILProtectedIndex` wrapper enclosing
 /// a `Python<'p>` marker and have it be the one implementing the
 /// `Graph` trait, but this would mean the `Graph` implementor would become
 /// likely to change between subsequent method invocations of the `hg-core`
 /// objects (a serious change of the `hg-core` API):
 /// either exposing ways to mutate the `Graph`, or making it a non persistent
 /// parameter in the relevant methods that need one.
 ///
 /// Another possibility would be to introduce an abstract lock handle into
 /// the core API, that would be tied to `GILGuard` / `Python<'p>`
 /// in the case of the `cpython` crate bindings yet could leave room for other
 /// mechanisms in other contexts.
 pub struct Index {
     index: PyObject,
     capi: &'static Revlog_CAPI,
 }
 
 impl Index {
     pub fn new(py: Python, index: PyObject) -> PyResult<Self> {
         let capi = unsafe { revlog_capi::retrieve(py)? };
         if capi.abi_version != REVLOG_CABI_VERSION {
             return Err(PyErr::new::<ImportError, _>(
                 py,
                 format!(
                     "ABI version mismatch: the C ABI revlog version {} \
                      does not match the {} expected by Rust hg-cpython",
                     capi.abi_version, REVLOG_CABI_VERSION
                 ),
             ));
         }
+        let compat: u64 = index.getattr(py, "rust_ext_compat")?.extract(py)?;
+        if compat == 0 {
+            return Err(PyErr::new::<TypeError, _>(
+                py,
+                "index object not compatible with Rust",
+            ));
+        }
         Ok(Index { index, capi })
     }
 
     /// return a reference to the CPython Index object in this Struct
     pub fn inner(&self) -> &PyObject {
         &self.index
     }
 
     pub fn append(&mut self, py: Python, tup: PyTuple) -> PyResult<PyObject> {
         self.index.call_method(
             py,
             "append",
             PyTuple::new(py, &[tup.into_object()]),
             None,
         )
     }
 }
 
 impl Clone for Index {
     fn clone(&self) -> Self {
         let guard = Python::acquire_gil();
         Index {
             index: self.index.clone_ref(guard.python()),
             capi: self.capi,
         }
     }
 }
 
 impl PyClone for Index {
     fn clone_ref(&self, py: Python) -> Self {
         Index {
             index: self.index.clone_ref(py),
             capi: self.capi,
         }
     }
 }
 
 impl Graph for Index {
     /// wrap a call to the C extern parents function
     fn parents(&self, rev: Revision) -> Result<[Revision; 2], GraphError> {
         if rev == WORKING_DIRECTORY_REVISION {
             return Err(GraphError::WorkingDirectoryUnsupported);
         }
         let mut res: [c_int; 2] = [0; 2];
         let code = unsafe {
             (self.capi.index_parents)(
                 self.index.as_ptr(),
                 rev as c_int,
                 &mut res as *mut [c_int; 2],
             )
         };
         match code {
             0 => Ok(res),
             _ => Err(GraphError::ParentOutOfRange(rev)),
         }
     }
 }
 
 impl RevlogIndex for Index {
     /// Note C return type is Py_ssize_t (hence signed), but we shall
     /// force it to unsigned, because it's a length
     fn len(&self) -> usize {
         unsafe { (self.capi.index_length)(self.index.as_ptr()) as usize }
     }
 
     fn node(&self, rev: Revision) -> Option<&Node> {
         let raw = unsafe {
             (self.capi.index_node)(self.index.as_ptr(), rev as ssize_t)
         };
         if raw.is_null() {
             None
         } else {
             // TODO it would be much better for the C layer to give us
             // a length, since the hash length will change in the near
             // future, but that's probably out of scope for the nodemap
             // patch series.
             //
             // The root of that unsafety relies in the signature of
             // `capi.index_node()` itself: returning a `Node` pointer
             // whereas it's a `char *` in the C counterpart.
             Some(unsafe { &*raw })
         }
     }
 }

rust/hg-cpython/src/revlog.rs

@@ -1,504 +1,509
 // revlog.rs
 //
 // Copyright 2019-2020 Georges Racinet <[e-mail]>
 //
 // This software may be used and distributed according to the terms of the
 // GNU General Public License version 2 or any later version.
 
 use crate::{
     cindex,
     utils::{node_from_py_bytes, node_from_py_object},
 };
 use cpython::{
     buffer::{Element, PyBuffer},
     exc::{IndexError, ValueError},
     ObjectProtocol, PyBytes, PyClone, PyDict, PyErr, PyInt, PyModule,
     PyObject, PyResult, PyString, PyTuple, Python, PythonObject, ToPyObject,
 };
 use hg::{
     nodemap::{Block, NodeMapError, NodeTree},
     revlog::{nodemap::NodeMap, NodePrefix, RevlogIndex},
     Revision,
 };
 use std::cell::RefCell;
 
 /// Return a Struct implementing the Graph trait
 pub(crate) fn pyindex_to_graph(
     py: Python,
     index: PyObject,
 ) -> PyResult<cindex::Index> {
     match index.extract::<MixedIndex>(py) {
         Ok(midx) => Ok(midx.clone_cindex(py)),
         Err(_) => cindex::Index::new(py, index),
     }
 }
 
 py_class!(pub class MixedIndex |py| {
     data cindex: RefCell<cindex::Index>;
     data nt: RefCell<Option<NodeTree>>;
     data docket: RefCell<Option<PyObject>>;
     // Holds a reference to the mmap'ed persistent nodemap data
     data mmap: RefCell<Option<PyBuffer>>;
 
     def __new__(_cls, cindex: PyObject) -> PyResult<MixedIndex> {
         Self::new(py, cindex)
     }
 
     /// Compatibility layer used for Python consumers needing access to the C index
     ///
     /// Only use case so far is `scmutil.shortesthexnodeidprefix`,
     /// that may need to build a custom `nodetree`, based on a specified revset.
     /// With a Rust implementation of the nodemap, we will be able to get rid of
     /// this, by exposing our own standalone nodemap class,
     /// ready to accept `MixedIndex`.
     def get_cindex(&self) -> PyResult<PyObject> {
         Ok(self.cindex(py).borrow().inner().clone_ref(py))
     }
 
     // Index API involving nodemap, as defined in mercurial/pure/parsers.py
 
     /// Return Revision if found, raises a bare `error.RevlogError`
     /// in case of ambiguity, same as C version does
     def get_rev(&self, node: PyBytes) -> PyResult<Option<Revision>> {
         let opt = self.get_nodetree(py)?.borrow();
         let nt = opt.as_ref().unwrap();
         let idx = &*self.cindex(py).borrow();
         let node = node_from_py_bytes(py, &node)?;
         nt.find_bin(idx, node.into()).map_err(|e| nodemap_error(py, e))
     }
 
     /// same as `get_rev()` but raises a bare `error.RevlogError` if node
     /// is not found.
     ///
     /// No need to repeat `node` in the exception, `mercurial/revlog.py`
     /// will catch and rewrap with it
     def rev(&self, node: PyBytes) -> PyResult<Revision> {
         self.get_rev(py, node)?.ok_or_else(|| revlog_error(py))
     }
 
     /// return True if the node exist in the index
     def has_node(&self, node: PyBytes) -> PyResult<bool> {
         self.get_rev(py, node).map(|opt| opt.is_some())
     }
 
     /// find length of shortest hex nodeid of a binary ID
     def shortest(&self, node: PyBytes) -> PyResult<usize> {
         let opt = self.get_nodetree(py)?.borrow();
         let nt = opt.as_ref().unwrap();
         let idx = &*self.cindex(py).borrow();
         match nt.unique_prefix_len_node(idx, &node_from_py_bytes(py, &node)?)
         {
             Ok(Some(l)) => Ok(l),
             Ok(None) => Err(revlog_error(py)),
             Err(e) => Err(nodemap_error(py, e)),
         }
     }
 
     def partialmatch(&self, node: PyObject) -> PyResult<Option<PyBytes>> {
         let opt = self.get_nodetree(py)?.borrow();
         let nt = opt.as_ref().unwrap();
         let idx = &*self.cindex(py).borrow();
 
         let node_as_string = if cfg!(feature = "python3-sys") {
             node.cast_as::<PyString>(py)?.to_string(py)?.to_string()
         }
         else {
             let node = node.extract::<PyBytes>(py)?;
             String::from_utf8_lossy(node.data(py)).to_string()
         };
 
         let prefix = NodePrefix::from_hex(&node_as_string).map_err(|_| PyErr::new::<ValueError, _>(py, "Invalid node or prefix"))?;
 
         nt.find_bin(idx, prefix)
             // TODO make an inner API returning the node directly
             .map(|opt| opt.map(
                 |rev| PyBytes::new(py, idx.node(rev).unwrap().as_bytes())))
             .map_err(|e| nodemap_error(py, e))
 
     }
 
     /// append an index entry
     def append(&self, tup: PyTuple) -> PyResult<PyObject> {
         if tup.len(py) < 8 {
             // this is better than the panic promised by tup.get_item()
             return Err(
                 PyErr::new::<IndexError, _>(py, "tuple index out of range"))
         }
         let node_bytes = tup.get_item(py, 7).extract(py)?;
         let node = node_from_py_object(py, &node_bytes)?;
 
         let mut idx = self.cindex(py).borrow_mut();
         let rev = idx.len() as Revision;
 
         idx.append(py, tup)?;
         self.get_nodetree(py)?.borrow_mut().as_mut().unwrap()
             .insert(&*idx, &node, rev)
             .map_err(|e| nodemap_error(py, e))?;
         Ok(py.None())
     }
 
     def __delitem__(&self, key: PyObject) -> PyResult<()> {
         // __delitem__ is both for `del idx[r]` and `del idx[r1:r2]`
         self.cindex(py).borrow().inner().del_item(py, key)?;
         let mut opt = self.get_nodetree(py)?.borrow_mut();
         let mut nt = opt.as_mut().unwrap();
         nt.invalidate_all();
         self.fill_nodemap(py, &mut nt)?;
         Ok(())
     }
 
     //
     // Reforwarded C index API
     //
 
     // index_methods (tp_methods). Same ordering as in revlog.c
 
     /// return the gca set of the given revs
     def ancestors(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "ancestors", args, kw)
     }
 
     /// return the heads of the common ancestors of the given revs
     def commonancestorsheads(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "commonancestorsheads", args, kw)
     }
 
     /// Clear the index caches and inner py_class data.
     /// It is Python's responsibility to call `update_nodemap_data` again.
     def clearcaches(&self, *args, **kw) -> PyResult<PyObject> {
         self.nt(py).borrow_mut().take();
         self.docket(py).borrow_mut().take();
         self.mmap(py).borrow_mut().take();
         self.call_cindex(py, "clearcaches", args, kw)
     }
 
     /// return the raw binary string representing a revision
     def entry_binary(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "entry_binary", args, kw)
     }
 
     /// return a binary packed version of the header
     def pack_header(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "pack_header", args, kw)
     }
 
     /// get an index entry
     def get(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "get", args, kw)
     }
 
     /// compute phases
     def computephasesmapsets(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "computephasesmapsets", args, kw)
     }
 
     /// reachableroots
     def reachableroots2(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "reachableroots2", args, kw)
     }
 
     /// get head revisions
     def headrevs(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "headrevs", args, kw)
     }
 
     /// get filtered head revisions
     def headrevsfiltered(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "headrevsfiltered", args, kw)
     }
 
     /// True if the object is a snapshot
     def issnapshot(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "issnapshot", args, kw)
     }
 
     /// Gather snapshot data in a cache dict
     def findsnapshots(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "findsnapshots", args, kw)
     }
 
     /// determine revisions with deltas to reconstruct fulltext
     def deltachain(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "deltachain", args, kw)
     }
 
     /// slice planned chunk read to reach a density threshold
     def slicechunktodensity(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "slicechunktodensity", args, kw)
     }
 
     /// stats for the index
     def stats(&self, *args, **kw) -> PyResult<PyObject> {
         self.call_cindex(py, "stats", args, kw)
     }
 
     // index_sequence_methods and index_mapping_methods.
     //
     // Since we call back through the high level Python API,
     // there's no point making a distinction between index_get
     // and index_getitem.
 
     def __len__(&self) -> PyResult<usize> {
         self.cindex(py).borrow().inner().len(py)
     }
 
     def __getitem__(&self, key: PyObject) -> PyResult<PyObject> {
         // this conversion seems needless, but that's actually because
         // `index_getitem` does not handle conversion from PyLong,
         // which expressions such as [e for e in index] internally use.
         // Note that we don't seem to have a direct way to call
         // PySequence_GetItem (does the job), which would possibly be better
         // for performance
         let key = match key.extract::<Revision>(py) {
             Ok(rev) => rev.to_py_object(py).into_object(),
             Err(_) => key,
         };
         self.cindex(py).borrow().inner().get_item(py, key)
     }
 
     def __setitem__(&self, key: PyObject, value: PyObject) -> PyResult<()> {
         self.cindex(py).borrow().inner().set_item(py, key, value)
     }
 
     def __contains__(&self, item: PyObject) -> PyResult<bool> {
         // ObjectProtocol does not seem to provide contains(), so
         // this is an equivalent implementation of the index_contains()
         // defined in revlog.c
         let cindex = self.cindex(py).borrow();
         match item.extract::<Revision>(py) {
             Ok(rev) => {
                 Ok(rev >= -1 && rev < cindex.inner().len(py)? as Revision)
             }
             Err(_) => {
                 cindex.inner().call_method(
                     py,
                     "has_node",
                     PyTuple::new(py, &[item]),
                     None)?
                 .extract(py)
             }
         }
     }
 
     def nodemap_data_all(&self) -> PyResult<PyBytes> {
         self.inner_nodemap_data_all(py)
     }
 
     def nodemap_data_incremental(&self) -> PyResult<PyObject> {
         self.inner_nodemap_data_incremental(py)
     }
     def update_nodemap_data(
         &self,
         docket: PyObject,
         nm_data: PyObject
     ) -> PyResult<PyObject> {
         self.inner_update_nodemap_data(py, docket, nm_data)
     }
 
     @property
     def entry_size(&self) -> PyResult<PyInt> {
         self.cindex(py).borrow().inner().getattr(py, "entry_size")?.extract::<PyInt>(py)
     }
 
+    @property
+    def rust_ext_compat(&self) -> PyResult<PyInt> {
+        self.cindex(py).borrow().inner().getattr(py, "rust_ext_compat")?.extract::<PyInt>(py)
+    }
+
 });
 
 impl MixedIndex {
     fn new(py: Python, cindex: PyObject) -> PyResult<MixedIndex> {
         Self::create_instance(
             py,
             RefCell::new(cindex::Index::new(py, cindex)?),
             RefCell::new(None),
             RefCell::new(None),
             RefCell::new(None),
         )
     }
 
     /// This is scaffolding at this point, but it could also become
     /// a way to start a persistent nodemap or perform a
     /// vacuum / repack operation
     fn fill_nodemap(
         &self,
         py: Python,
         nt: &mut NodeTree,
     ) -> PyResult<PyObject> {
         let index = self.cindex(py).borrow();
         for r in 0..index.len() {
             let rev = r as Revision;
             // in this case node() won't ever return None
             nt.insert(&*index, index.node(rev).unwrap(), rev)
                 .map_err(|e| nodemap_error(py, e))?
         }
         Ok(py.None())
     }
 
     fn get_nodetree<'a>(
         &'a self,
         py: Python<'a>,
     ) -> PyResult<&'a RefCell<Option<NodeTree>>> {
         if self.nt(py).borrow().is_none() {
             let readonly = Box::new(Vec::new());
             let mut nt = NodeTree::load_bytes(readonly, 0);
             self.fill_nodemap(py, &mut nt)?;
             self.nt(py).borrow_mut().replace(nt);
         }
         Ok(self.nt(py))
     }
 
     /// forward a method call to the underlying C index
     fn call_cindex(
         &self,
         py: Python,
         name: &str,
         args: &PyTuple,
         kwargs: Option<&PyDict>,
     ) -> PyResult<PyObject> {
         self.cindex(py)
             .borrow()
             .inner()
             .call_method(py, name, args, kwargs)
     }
 
     pub fn clone_cindex(&self, py: Python) -> cindex::Index {
         self.cindex(py).borrow().clone_ref(py)
     }
 
     /// Returns the full nodemap bytes to be written as-is to disk
     fn inner_nodemap_data_all(&self, py: Python) -> PyResult<PyBytes> {
         let nodemap = self.get_nodetree(py)?.borrow_mut().take().unwrap();
         let (readonly, bytes) = nodemap.into_readonly_and_added_bytes();
 
         // If there's anything readonly, we need to build the data again from
         // scratch
         let bytes = if readonly.len() > 0 {
             let mut nt = NodeTree::load_bytes(Box::new(vec![]), 0);
             self.fill_nodemap(py, &mut nt)?;
 
             let (readonly, bytes) = nt.into_readonly_and_added_bytes();
             assert_eq!(readonly.len(), 0);
 
             bytes
         } else {
             bytes
         };
 
         let bytes = PyBytes::new(py, &bytes);
         Ok(bytes)
     }
 
     /// Returns the last saved docket along with the size of any changed data
     /// (in number of blocks), and said data as bytes.
     fn inner_nodemap_data_incremental(
         &self,
         py: Python,
     ) -> PyResult<PyObject> {
         let docket = self.docket(py).borrow();
         let docket = match docket.as_ref() {
             Some(d) => d,
             None => return Ok(py.None()),
         };
 
         let node_tree = self.get_nodetree(py)?.borrow_mut().take().unwrap();
         let masked_blocks = node_tree.masked_readonly_blocks();
         let (_, data) = node_tree.into_readonly_and_added_bytes();
         let changed = masked_blocks * std::mem::size_of::<Block>();
 
         Ok((docket, changed, PyBytes::new(py, &data))
             .to_py_object(py)
             .into_object())
     }
 
     /// Update the nodemap from the new (mmaped) data.
     /// The docket is kept as a reference for later incremental calls.
     fn inner_update_nodemap_data(
         &self,
         py: Python,
         docket: PyObject,
         nm_data: PyObject,
     ) -> PyResult<PyObject> {
         let buf = PyBuffer::get(py, &nm_data)?;
         let len = buf.item_count();
 
         // Build a slice from the mmap'ed buffer data
         let cbuf = buf.buf_ptr();
         let bytes = if std::mem::size_of::<u8>() == buf.item_size()
             && buf.is_c_contiguous()
             && u8::is_compatible_format(buf.format())
         {
             unsafe { std::slice::from_raw_parts(cbuf as *const u8, len) }
         } else {
             return Err(PyErr::new::<ValueError, _>(
                 py,
                 "Nodemap data buffer has an invalid memory representation"
                     .to_string(),
             ));
         };
 
         // Keep a reference to the mmap'ed buffer, otherwise we get a dangling
         // pointer.
         self.mmap(py).borrow_mut().replace(buf);
 
         let mut nt = NodeTree::load_bytes(Box::new(bytes), len);
 
         let data_tip =
             docket.getattr(py, "tip_rev")?.extract::<Revision>(py)?;
         self.docket(py).borrow_mut().replace(docket.clone_ref(py));
         let idx = self.cindex(py).borrow();
         let current_tip = idx.len();
 
         for r in (data_tip + 1)..current_tip as Revision {
             let rev = r as Revision;
             // in this case node() won't ever return None
             nt.insert(&*idx, idx.node(rev).unwrap(), rev)
                 .map_err(|e| nodemap_error(py, e))?
         }
 
         *self.nt(py).borrow_mut() = Some(nt);
 
         Ok(py.None())
     }
 }
 
 fn revlog_error(py: Python) -> PyErr {
     match py
         .import("mercurial.error")
         .and_then(|m| m.get(py, "RevlogError"))
     {
         Err(e) => e,
         Ok(cls) => PyErr::from_instance(py, cls),
     }
 }
 
 fn rev_not_in_index(py: Python, rev: Revision) -> PyErr {
     PyErr::new::<ValueError, _>(
         py,
         format!(
             "Inconsistency: Revision {} found in nodemap \
              is not in revlog index",
             rev
         ),
     )
 }
 
 /// Standard treatment of NodeMapError
 fn nodemap_error(py: Python, err: NodeMapError) -> PyErr {
     match err {
         NodeMapError::MultipleResults => revlog_error(py),
         NodeMapError::RevisionNotInIndex(r) => rev_not_in_index(py, r),
     }
 }
 
 /// Create the module, with __package__ given from parent
 pub fn init_module(py: Python, package: &str) -> PyResult<PyModule> {
     let dotted_name = &format!("{}.revlog", package);
     let m = PyModule::new(py, dotted_name)?;
     m.add(py, "__package__", package)?;
     m.add(py, "__doc__", "RevLog - Rust implementations")?;
 
     m.add_class::<MixedIndex>(py)?;
 
     let sys = PyModule::import(py, "sys")?;
     let sys_modules: PyDict = sys.get(py, "modules")?.extract(py)?;
     sys_modules.set_item(py, dotted_name, &m)?;
 
     Ok(m)
 }