cext: split character encoding functions to new compilation unit

This extracts charencode.c from parsers.c, which seems big enough for me
to hesitate to add new JSON functions. Still charencode.o is linked to
parsers.so to avoid duplication of binary codes.

mercurial/cext/charencode.c

@@ -1,997 +1,209
 /*
- parsers.c - efficient content parsing
+ charencode.c - miscellaneous character encoding
 
  Copyright 2008 Matt Mackall <[e-mail]> and others
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 */
 
 #include <Python.h>
-#include <ctype.h>
-#include <stddef.h>
-#include <string.h>
 
 #include "util.h"
-#include "bitmanipulation.h"
-
-#ifdef IS_PY3K
-/* The mapping of Python types is meant to be temporary to get Python
- * 3 to compile. We should remove this once Python 3 support is fully
- * supported and proper types are used in the extensions themselves. */
-#define PyInt_Type PyLong_Type
-#define PyInt_Check PyLong_Check
-#define PyInt_FromLong PyLong_FromLong
-#define PyInt_FromSsize_t PyLong_FromSsize_t
-#define PyInt_AS_LONG PyLong_AS_LONG
-#define PyInt_AsLong PyLong_AsLong
-#endif
-
-static const char *const versionerrortext = "Python minor version mismatch";
 
 static const char lowertable[128] = {
 	'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07',
 	'\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
 	'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17',
 	'\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
 	'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27',
 	'\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
 	'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37',
 	'\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
 	'\x40',
 	        '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67', /* A-G */
 	'\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f', /* H-O */
 	'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77', /* P-W */
 	'\x78', '\x79', '\x7a',                                         /* X-Z */
 	                        '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
 	'\x60', '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67',
 	'\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f',
 	'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77',
 	'\x78', '\x79', '\x7a', '\x7b', '\x7c', '\x7d', '\x7e', '\x7f'
 };
 
 static const char uppertable[128] = {
 	'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07',
 	'\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
 	'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17',
 	'\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
 	'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27',
 	'\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
 	'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37',
 	'\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
 	'\x40', '\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47',
 	'\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f',
 	'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57',
 	'\x58', '\x59', '\x5a', '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
 	'\x60',
 		'\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47', /* a-g */
 	'\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f', /* h-o */
 	'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57', /* p-w */
 	'\x58', '\x59', '\x5a', 					/* x-z */
 				'\x7b', '\x7c', '\x7d', '\x7e', '\x7f'
 };
 
 /*
  * Turn a hex-encoded string into binary.
  */
 PyObject *unhexlify(const char *str, int len)
 {
 	PyObject *ret;
 	char *d;
 	int i;
 
 	ret = PyBytes_FromStringAndSize(NULL, len / 2);
 
 	if (!ret)
 		return NULL;
 
 	d = PyBytes_AsString(ret);
 
 	for (i = 0; i < len;) {
 		int hi = hexdigit(str, i++);
 		int lo = hexdigit(str, i++);
 		*d++ = (hi << 4) | lo;
 	}
 
 	return ret;
 }
 
 static inline PyObject *_asciitransform(PyObject *str_obj,
 					const char table[128],
 					PyObject *fallback_fn)
 {
 	char *str, *newstr;
 	Py_ssize_t i, len;
 	PyObject *newobj = NULL;
 	PyObject *ret = NULL;
 
 	str = PyBytes_AS_STRING(str_obj);
 	len = PyBytes_GET_SIZE(str_obj);
 
 	newobj = PyBytes_FromStringAndSize(NULL, len);
 	if (!newobj)
 		goto quit;
 
 	newstr = PyBytes_AS_STRING(newobj);
 
 	for (i = 0; i < len; i++) {
 		char c = str[i];
 		if (c & 0x80) {
 			if (fallback_fn != NULL) {
 				ret = PyObject_CallFunctionObjArgs(fallback_fn,
 					str_obj, NULL);
 			} else {
 				PyObject *err = PyUnicodeDecodeError_Create(
 					"ascii", str, len, i, (i + 1),
 					"unexpected code byte");
 				PyErr_SetObject(PyExc_UnicodeDecodeError, err);
 				Py_XDECREF(err);
 			}
 			goto quit;
 		}
 		newstr[i] = table[(unsigned char)c];
 	}
 
 	ret = newobj;
 	Py_INCREF(ret);
 quit:
 	Py_XDECREF(newobj);
 	return ret;
 }
 
-static PyObject *asciilower(PyObject *self, PyObject *args)
+PyObject *asciilower(PyObject *self, PyObject *args)
 {
 	PyObject *str_obj;
 	if (!PyArg_ParseTuple(args, "O!:asciilower", &PyBytes_Type, &str_obj))
 		return NULL;
 	return _asciitransform(str_obj, lowertable, NULL);
 }
 
-static PyObject *asciiupper(PyObject *self, PyObject *args)
+PyObject *asciiupper(PyObject *self, PyObject *args)
 {
 	PyObject *str_obj;
 	if (!PyArg_ParseTuple(args, "O!:asciiupper", &PyBytes_Type, &str_obj))
 		return NULL;
 	return _asciitransform(str_obj, uppertable, NULL);
 }
 
-static PyObject *dict_new_presized(PyObject *self, PyObject *args)
-{
-	Py_ssize_t expected_size;
-
-	if (!PyArg_ParseTuple(args, "n:make_presized_dict", &expected_size))
-		return NULL;
-
-	return _dict_new_presized(expected_size);
-}
-
-static PyObject *make_file_foldmap(PyObject *self, PyObject *args)
+PyObject *make_file_foldmap(PyObject *self, PyObject *args)
 {
 	PyObject *dmap, *spec_obj, *normcase_fallback;
 	PyObject *file_foldmap = NULL;
 	enum normcase_spec spec;
 	PyObject *k, *v;
 	dirstateTupleObject *tuple;
 	Py_ssize_t pos = 0;
 	const char *table;
 
 	if (!PyArg_ParseTuple(args, "O!O!O!:make_file_foldmap",
 			      &PyDict_Type, &dmap,
 			      &PyInt_Type, &spec_obj,
 			      &PyFunction_Type, &normcase_fallback))
 		goto quit;
 
 	spec = (int)PyInt_AS_LONG(spec_obj);
 	switch (spec) {
 	case NORMCASE_LOWER:
 		table = lowertable;
 		break;
 	case NORMCASE_UPPER:
 		table = uppertable;
 		break;
 	case NORMCASE_OTHER:
 		table = NULL;
 		break;
 	default:
 		PyErr_SetString(PyExc_TypeError, "invalid normcasespec");
 		goto quit;
 	}
 
 	/* Add some more entries to deal with additions outside this
 	   function. */
 	file_foldmap = _dict_new_presized((PyDict_Size(dmap) / 10) * 11);
 	if (file_foldmap == NULL)
 		goto quit;
 
 	while (PyDict_Next(dmap, &pos, &k, &v)) {
 		if (!dirstate_tuple_check(v)) {
 			PyErr_SetString(PyExc_TypeError,
 					"expected a dirstate tuple");
 			goto quit;
 		}
 
 		tuple = (dirstateTupleObject *)v;
 		if (tuple->state != 'r') {
 			PyObject *normed;
 			if (table != NULL) {
 				normed = _asciitransform(k, table,
 					normcase_fallback);
 			} else {
 				normed = PyObject_CallFunctionObjArgs(
 					normcase_fallback, k, NULL);
 			}
 
 			if (normed == NULL)
 				goto quit;
 			if (PyDict_SetItem(file_foldmap, normed, k) == -1) {
 				Py_DECREF(normed);
 				goto quit;
 			}
 			Py_DECREF(normed);
 		}
 	}
 	return file_foldmap;
 quit:
 	Py_XDECREF(file_foldmap);
 	return NULL;
 }
-
-/*
- * This code assumes that a manifest is stitched together with newline
- * ('\n') characters.
- */
-static PyObject *parse_manifest(PyObject *self, PyObject *args)
-{
-	PyObject *mfdict, *fdict;
-	char *str, *start, *end;
-	int len;
-
-	if (!PyArg_ParseTuple(args, "O!O!s#:parse_manifest",
-			      &PyDict_Type, &mfdict,
-			      &PyDict_Type, &fdict,
-			      &str, &len))
-		goto quit;
-
-	start = str;
-	end = str + len;
-	while (start < end) {
-		PyObject *file = NULL, *node = NULL;
-		PyObject *flags = NULL;
-		char *zero = NULL, *newline = NULL;
-		ptrdiff_t nlen;
-
-		zero = memchr(start, '\0', end - start);
-		if (!zero) {
-			PyErr_SetString(PyExc_ValueError,
-					"manifest entry has no separator");
-			goto quit;
-		}
-
-		newline = memchr(zero + 1, '\n', end - (zero + 1));
-		if (!newline) {
-			PyErr_SetString(PyExc_ValueError,
-					"manifest contains trailing garbage");
-			goto quit;
-		}
-
-		file = PyBytes_FromStringAndSize(start, zero - start);
-
-		if (!file)
-			goto bail;
-
-		nlen = newline - zero - 1;
-
-		node = unhexlify(zero + 1, nlen > 40 ? 40 : (int)nlen);
-		if (!node)
-			goto bail;
-
-		if (nlen > 40) {
-			flags = PyBytes_FromStringAndSize(zero + 41,
-							   nlen - 40);
-			if (!flags)
-				goto bail;
-
-			if (PyDict_SetItem(fdict, file, flags) == -1)
-				goto bail;
-		}
-
-		if (PyDict_SetItem(mfdict, file, node) == -1)
-			goto bail;
-
-		start = newline + 1;
-
-		Py_XDECREF(flags);
-		Py_XDECREF(node);
-		Py_XDECREF(file);
-		continue;
-	bail:
-		Py_XDECREF(flags);
-		Py_XDECREF(node);
-		Py_XDECREF(file);
-		goto quit;
-	}
-
-	Py_INCREF(Py_None);
-	return Py_None;
-quit:
-	return NULL;
-}
-
-static inline dirstateTupleObject *make_dirstate_tuple(char state, int mode,
-						       int size, int mtime)
-{
-	dirstateTupleObject *t = PyObject_New(dirstateTupleObject,
-					      &dirstateTupleType);
-	if (!t)
-		return NULL;
-	t->state = state;
-	t->mode = mode;
-	t->size = size;
-	t->mtime = mtime;
-	return t;
-}
-
-static PyObject *dirstate_tuple_new(PyTypeObject *subtype, PyObject *args,
-				    PyObject *kwds)
-{
-	/* We do all the initialization here and not a tp_init function because
-	 * dirstate_tuple is immutable. */
-	dirstateTupleObject *t;
-	char state;
-	int size, mode, mtime;
-	if (!PyArg_ParseTuple(args, "ciii", &state, &mode, &size, &mtime))
-		return NULL;
-
-	t = (dirstateTupleObject *)subtype->tp_alloc(subtype, 1);
-	if (!t)
-		return NULL;
-	t->state = state;
-	t->mode = mode;
-	t->size = size;
-	t->mtime = mtime;
-
-	return (PyObject *)t;
-}
-
-static void dirstate_tuple_dealloc(PyObject *o)
-{
-	PyObject_Del(o);
-}
-
-static Py_ssize_t dirstate_tuple_length(PyObject *o)
-{
-	return 4;
-}
-
-static PyObject *dirstate_tuple_item(PyObject *o, Py_ssize_t i)
-{
-	dirstateTupleObject *t = (dirstateTupleObject *)o;
-	switch (i) {
-	case 0:
-		return PyBytes_FromStringAndSize(&t->state, 1);
-	case 1:
-		return PyInt_FromLong(t->mode);
-	case 2:
-		return PyInt_FromLong(t->size);
-	case 3:
-		return PyInt_FromLong(t->mtime);
-	default:
-		PyErr_SetString(PyExc_IndexError, "index out of range");
-		return NULL;
-	}
-}
-
-static PySequenceMethods dirstate_tuple_sq = {
-	dirstate_tuple_length,     /* sq_length */
-	0,                         /* sq_concat */
-	0,                         /* sq_repeat */
-	dirstate_tuple_item,       /* sq_item */
-	0,                         /* sq_ass_item */
-	0,                         /* sq_contains */
-	0,                         /* sq_inplace_concat */
-	0                          /* sq_inplace_repeat */
-};
-
-PyTypeObject dirstateTupleType = {
-	PyVarObject_HEAD_INIT(NULL, 0)
-	"dirstate_tuple",          /* tp_name */
-	sizeof(dirstateTupleObject),/* tp_basicsize */
-	0,                         /* tp_itemsize */
-	(destructor)dirstate_tuple_dealloc, /* tp_dealloc */
-	0,                         /* tp_print */
-	0,                         /* tp_getattr */
-	0,                         /* tp_setattr */
-	0,                         /* tp_compare */
-	0,                         /* tp_repr */
-	0,                         /* tp_as_number */
-	&dirstate_tuple_sq,        /* tp_as_sequence */
-	0,                         /* tp_as_mapping */
-	0,                         /* tp_hash  */
-	0,                         /* tp_call */
-	0,                         /* tp_str */
-	0,                         /* tp_getattro */
-	0,                         /* tp_setattro */
-	0,                         /* tp_as_buffer */
-	Py_TPFLAGS_DEFAULT,        /* tp_flags */
-	"dirstate tuple",          /* tp_doc */
-	0,                         /* tp_traverse */
-	0,                         /* tp_clear */
-	0,                         /* tp_richcompare */
-	0,                         /* tp_weaklistoffset */
-	0,                         /* tp_iter */
-	0,                         /* tp_iternext */
-	0,                         /* tp_methods */
-	0,                         /* tp_members */
-	0,                         /* tp_getset */
-	0,                         /* tp_base */
-	0,                         /* tp_dict */
-	0,                         /* tp_descr_get */
-	0,                         /* tp_descr_set */
-	0,                         /* tp_dictoffset */
-	0,                         /* tp_init */
-	0,                         /* tp_alloc */
-	dirstate_tuple_new,        /* tp_new */
-};
-
-static PyObject *parse_dirstate(PyObject *self, PyObject *args)
-{
-	PyObject *dmap, *cmap, *parents = NULL, *ret = NULL;
-	PyObject *fname = NULL, *cname = NULL, *entry = NULL;
-	char state, *cur, *str, *cpos;
-	int mode, size, mtime;
-	unsigned int flen, len, pos = 40;
-	int readlen;
-
-	if (!PyArg_ParseTuple(args, "O!O!s#:parse_dirstate",
-			      &PyDict_Type, &dmap,
-			      &PyDict_Type, &cmap,
-			      &str, &readlen))
-		goto quit;
-
-	len = readlen;
-
-	/* read parents */
-	if (len < 40) {
-		PyErr_SetString(
-			PyExc_ValueError, "too little data for parents");
-		goto quit;
-	}
-
-	parents = Py_BuildValue("s#s#", str, 20, str + 20, 20);
-	if (!parents)
-		goto quit;
-
-	/* read filenames */
-	while (pos >= 40 && pos < len) {
-		if (pos + 17 > len) {
-			PyErr_SetString(PyExc_ValueError,
-					"overflow in dirstate");
-			goto quit;
-		}
-		cur = str + pos;
-		/* unpack header */
-		state = *cur;
-		mode = getbe32(cur + 1);
-		size = getbe32(cur + 5);
-		mtime = getbe32(cur + 9);
-		flen = getbe32(cur + 13);
-		pos += 17;
-		cur += 17;
-		if (flen > len - pos) {
-			PyErr_SetString(PyExc_ValueError, "overflow in dirstate");
-			goto quit;
-		}
-
-		entry = (PyObject *)make_dirstate_tuple(state, mode, size,
-							mtime);
-		cpos = memchr(cur, 0, flen);
-		if (cpos) {
-			fname = PyBytes_FromStringAndSize(cur, cpos - cur);
-			cname = PyBytes_FromStringAndSize(cpos + 1,
-							   flen - (cpos - cur) - 1);
-			if (!fname || !cname ||
-			    PyDict_SetItem(cmap, fname, cname) == -1 ||
-			    PyDict_SetItem(dmap, fname, entry) == -1)
-				goto quit;
-			Py_DECREF(cname);
-		} else {
-			fname = PyBytes_FromStringAndSize(cur, flen);
-			if (!fname ||
-			    PyDict_SetItem(dmap, fname, entry) == -1)
-				goto quit;
-		}
-		Py_DECREF(fname);
-		Py_DECREF(entry);
-		fname = cname = entry = NULL;
-		pos += flen;
-	}
-
-	ret = parents;
-	Py_INCREF(ret);
-quit:
-	Py_XDECREF(fname);
-	Py_XDECREF(cname);
-	Py_XDECREF(entry);
-	Py_XDECREF(parents);
-	return ret;
-}
-
-/*
- * Build a set of non-normal and other parent entries from the dirstate dmap
-*/
-static PyObject *nonnormalotherparententries(PyObject *self, PyObject *args) {
-	PyObject *dmap, *fname, *v;
-	PyObject *nonnset = NULL, *otherpset = NULL, *result = NULL;
-	Py_ssize_t pos;
-
-	if (!PyArg_ParseTuple(args, "O!:nonnormalentries",
-			      &PyDict_Type, &dmap))
-		goto bail;
-
-	nonnset = PySet_New(NULL);
-	if (nonnset == NULL)
-		goto bail;
-
-	otherpset = PySet_New(NULL);
-	if (otherpset == NULL)
-		goto bail;
-
-	pos = 0;
-	while (PyDict_Next(dmap, &pos, &fname, &v)) {
-		dirstateTupleObject *t;
-		if (!dirstate_tuple_check(v)) {
-			PyErr_SetString(PyExc_TypeError,
-					"expected a dirstate tuple");
-			goto bail;
-		}
-		t = (dirstateTupleObject *)v;
-
-		if (t->state == 'n' && t->size == -2) {
-			if (PySet_Add(otherpset, fname) == -1) {
-				goto bail;
-			}
-		}
-
-		if (t->state == 'n' && t->mtime != -1)
-			continue;
-		if (PySet_Add(nonnset, fname) == -1)
-			goto bail;
-	}
-
-	result = Py_BuildValue("(OO)", nonnset, otherpset);
-	if (result == NULL)
-		goto bail;
-	Py_DECREF(nonnset);
-	Py_DECREF(otherpset);
-	return result;
-bail:
-	Py_XDECREF(nonnset);
-	Py_XDECREF(otherpset);
-	Py_XDECREF(result);
-	return NULL;
-}
-
-/*
- * Efficiently pack a dirstate object into its on-disk format.
- */
-static PyObject *pack_dirstate(PyObject *self, PyObject *args)
-{
-	PyObject *packobj = NULL;
-	PyObject *map, *copymap, *pl, *mtime_unset = NULL;
-	Py_ssize_t nbytes, pos, l;
-	PyObject *k, *v = NULL, *pn;
-	char *p, *s;
-	int now;
-
-	if (!PyArg_ParseTuple(args, "O!O!Oi:pack_dirstate",
-			      &PyDict_Type, &map, &PyDict_Type, &copymap,
-			      &pl, &now))
-		return NULL;
-
-	if (!PySequence_Check(pl) || PySequence_Size(pl) != 2) {
-		PyErr_SetString(PyExc_TypeError, "expected 2-element sequence");
-		return NULL;
-	}
-
-	/* Figure out how much we need to allocate. */
-	for (nbytes = 40, pos = 0; PyDict_Next(map, &pos, &k, &v);) {
-		PyObject *c;
-		if (!PyBytes_Check(k)) {
-			PyErr_SetString(PyExc_TypeError, "expected string key");
-			goto bail;
-		}
-		nbytes += PyBytes_GET_SIZE(k) + 17;
-		c = PyDict_GetItem(copymap, k);
-		if (c) {
-			if (!PyBytes_Check(c)) {
-				PyErr_SetString(PyExc_TypeError,
-						"expected string key");
-				goto bail;
-			}
-			nbytes += PyBytes_GET_SIZE(c) + 1;
-		}
-	}
-
-	packobj = PyBytes_FromStringAndSize(NULL, nbytes);
-	if (packobj == NULL)
-		goto bail;
-
-	p = PyBytes_AS_STRING(packobj);
-
-	pn = PySequence_ITEM(pl, 0);
-	if (PyBytes_AsStringAndSize(pn, &s, &l) == -1 || l != 20) {
-		PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash");
-		goto bail;
-	}
-	memcpy(p, s, l);
-	p += 20;
-	pn = PySequence_ITEM(pl, 1);
-	if (PyBytes_AsStringAndSize(pn, &s, &l) == -1 || l != 20) {
-		PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash");
-		goto bail;
-	}
-	memcpy(p, s, l);
-	p += 20;
-
-	for (pos = 0; PyDict_Next(map, &pos, &k, &v); ) {
-		dirstateTupleObject *tuple;
-		char state;
-		int mode, size, mtime;
-		Py_ssize_t len, l;
-		PyObject *o;
-		char *t;
-
-		if (!dirstate_tuple_check(v)) {
-			PyErr_SetString(PyExc_TypeError,
-					"expected a dirstate tuple");
-			goto bail;
-		}
-		tuple = (dirstateTupleObject *)v;
-
-		state = tuple->state;
-		mode = tuple->mode;
-		size = tuple->size;
-		mtime = tuple->mtime;
-		if (state == 'n' && mtime == now) {
-			/* See pure/parsers.py:pack_dirstate for why we do
-			 * this. */
-			mtime = -1;
-			mtime_unset = (PyObject *)make_dirstate_tuple(
-				state, mode, size, mtime);
-			if (!mtime_unset)
-				goto bail;
-			if (PyDict_SetItem(map, k, mtime_unset) == -1)
-				goto bail;
-			Py_DECREF(mtime_unset);
-			mtime_unset = NULL;
-		}
-		*p++ = state;
-		putbe32((uint32_t)mode, p);
-		putbe32((uint32_t)size, p + 4);
-		putbe32((uint32_t)mtime, p + 8);
-		t = p + 12;
-		p += 16;
-		len = PyBytes_GET_SIZE(k);
-		memcpy(p, PyBytes_AS_STRING(k), len);
-		p += len;
-		o = PyDict_GetItem(copymap, k);
-		if (o) {
-			*p++ = '\0';
-			l = PyBytes_GET_SIZE(o);
-			memcpy(p, PyBytes_AS_STRING(o), l);
-			p += l;
-			len += l + 1;
-		}
-		putbe32((uint32_t)len, t);
-	}
-
-	pos = p - PyBytes_AS_STRING(packobj);
-	if (pos != nbytes) {
-		PyErr_Format(PyExc_SystemError, "bad dirstate size: %ld != %ld",
-                             (long)pos, (long)nbytes);
-		goto bail;
-	}
-
-	return packobj;
-bail:
-	Py_XDECREF(mtime_unset);
-	Py_XDECREF(packobj);
-	Py_XDECREF(v);
-	return NULL;
-}
-
-#define BUMPED_FIX 1
-#define USING_SHA_256 2
-#define FM1_HEADER_SIZE (4 + 8 + 2 + 2 + 1 + 1 + 1)
-
-static PyObject *readshas(
-	const char *source, unsigned char num, Py_ssize_t hashwidth)
-{
-	int i;
-	PyObject *list = PyTuple_New(num);
-	if (list == NULL) {
-		return NULL;
-	}
-	for (i = 0; i < num; i++) {
-		PyObject *hash = PyBytes_FromStringAndSize(source, hashwidth);
-		if (hash == NULL) {
-			Py_DECREF(list);
-			return NULL;
-		}
-		PyTuple_SET_ITEM(list, i, hash);
-		source += hashwidth;
-	}
-	return list;
-}
-
-static PyObject *fm1readmarker(const char *databegin, const char *dataend,
-			       uint32_t *msize)
-{
-	const char *data = databegin;
-	const char *meta;
-
-	double mtime;
-	int16_t tz;
-	uint16_t flags;
-	unsigned char nsuccs, nparents, nmetadata;
-	Py_ssize_t hashwidth = 20;
-
-	PyObject *prec = NULL, *parents = NULL, *succs = NULL;
-	PyObject *metadata = NULL, *ret = NULL;
-	int i;
-
-	if (data + FM1_HEADER_SIZE > dataend) {
-		goto overflow;
-	}
-
-	*msize = getbe32(data);
-	data += 4;
-	mtime = getbefloat64(data);
-	data += 8;
-	tz = getbeint16(data);
-	data += 2;
-	flags = getbeuint16(data);
-	data += 2;
-
-	if (flags & USING_SHA_256) {
-		hashwidth = 32;
-	}
-
-	nsuccs = (unsigned char)(*data++);
-	nparents = (unsigned char)(*data++);
-	nmetadata = (unsigned char)(*data++);
-
-	if (databegin + *msize > dataend) {
-		goto overflow;
-	}
-	dataend = databegin + *msize;  /* narrow down to marker size */
-
-	if (data + hashwidth > dataend) {
-		goto overflow;
-	}
-	prec = PyBytes_FromStringAndSize(data, hashwidth);
-	data += hashwidth;
-	if (prec == NULL) {
-		goto bail;
-	}
-
-	if (data + nsuccs * hashwidth > dataend) {
-		goto overflow;
-	}
-	succs = readshas(data, nsuccs, hashwidth);
-	if (succs == NULL) {
-		goto bail;
-	}
-	data += nsuccs * hashwidth;
-
-	if (nparents == 1 || nparents == 2) {
-		if (data + nparents * hashwidth > dataend) {
-			goto overflow;
-		}
-		parents = readshas(data, nparents, hashwidth);
-		if (parents == NULL) {
-			goto bail;
-		}
-		data += nparents * hashwidth;
-	} else {
-		parents = Py_None;
-		Py_INCREF(parents);
-	}
-
-	if (data + 2 * nmetadata > dataend) {
-		goto overflow;
-	}
-	meta = data + (2 * nmetadata);
-	metadata = PyTuple_New(nmetadata);
-	if (metadata == NULL) {
-		goto bail;
-	}
-	for (i = 0; i < nmetadata; i++) {
-		PyObject *tmp, *left = NULL, *right = NULL;
-		Py_ssize_t leftsize = (unsigned char)(*data++);
-		Py_ssize_t rightsize = (unsigned char)(*data++);
-		if (meta + leftsize + rightsize > dataend) {
-			goto overflow;
-		}
-		left = PyBytes_FromStringAndSize(meta, leftsize);
-		meta += leftsize;
-		right = PyBytes_FromStringAndSize(meta, rightsize);
-		meta += rightsize;
-		tmp = PyTuple_New(2);
-		if (!left || !right || !tmp) {
-			Py_XDECREF(left);
-			Py_XDECREF(right);
-			Py_XDECREF(tmp);
-			goto bail;
-		}
-		PyTuple_SET_ITEM(tmp, 0, left);
-		PyTuple_SET_ITEM(tmp, 1, right);
-		PyTuple_SET_ITEM(metadata, i, tmp);
-	}
-	ret = Py_BuildValue("(OOHO(di)O)", prec, succs, flags,
-			    metadata, mtime, (int)tz * 60, parents);
-	goto bail;  /* return successfully */
-
-overflow:
-	PyErr_SetString(PyExc_ValueError, "overflow in obsstore");
-bail:
-	Py_XDECREF(prec);
-	Py_XDECREF(succs);
-	Py_XDECREF(metadata);
-	Py_XDECREF(parents);
-	return ret;
-}
-
-
-static PyObject *fm1readmarkers(PyObject *self, PyObject *args) {
-	const char *data, *dataend;
-	int datalen;
-	Py_ssize_t offset, stop;
-	PyObject *markers = NULL;
-
-	if (!PyArg_ParseTuple(args, "s#nn", &data, &datalen, &offset, &stop)) {
-		return NULL;
-	}
-	dataend = data + datalen;
-	data += offset;
-	markers = PyList_New(0);
-	if (!markers) {
-		return NULL;
-	}
-	while (offset < stop) {
-		uint32_t msize;
-		int error;
-		PyObject *record = fm1readmarker(data, dataend, &msize);
-		if (!record) {
-			goto bail;
-		}
-		error = PyList_Append(markers, record);
-		Py_DECREF(record);
-		if (error) {
-			goto bail;
-		}
-		data += msize;
-		offset += msize;
-	}
-	return markers;
-bail:
-	Py_DECREF(markers);
-	return NULL;
-}
-
-static char parsers_doc[] = "Efficient content parsing.";
-
-PyObject *encodedir(PyObject *self, PyObject *args);
-PyObject *pathencode(PyObject *self, PyObject *args);
-PyObject *lowerencode(PyObject *self, PyObject *args);
-PyObject *parse_index2(PyObject *self, PyObject *args);
-
-static PyMethodDef methods[] = {
-	{"pack_dirstate", pack_dirstate, METH_VARARGS, "pack a dirstate\n"},
-	{"nonnormalotherparententries", nonnormalotherparententries, METH_VARARGS,
-	"create a set containing non-normal and other parent entries of given "
-	"dirstate\n"},
-	{"parse_manifest", parse_manifest, METH_VARARGS, "parse a manifest\n"},
-	{"parse_dirstate", parse_dirstate, METH_VARARGS, "parse a dirstate\n"},
-	{"parse_index2", parse_index2, METH_VARARGS, "parse a revlog index\n"},
-	{"asciilower", asciilower, METH_VARARGS, "lowercase an ASCII string\n"},
-	{"asciiupper", asciiupper, METH_VARARGS, "uppercase an ASCII string\n"},
-	{"dict_new_presized", dict_new_presized, METH_VARARGS,
-	 "construct a dict with an expected size\n"},
-	{"make_file_foldmap", make_file_foldmap, METH_VARARGS,
-	 "make file foldmap\n"},
-	{"encodedir", encodedir, METH_VARARGS, "encodedir a path\n"},
-	{"pathencode", pathencode, METH_VARARGS, "fncache-encode a path\n"},
-	{"lowerencode", lowerencode, METH_VARARGS, "lower-encode a path\n"},
-	{"fm1readmarkers", fm1readmarkers, METH_VARARGS,
-			"parse v1 obsolete markers\n"},
-	{NULL, NULL}
-};
-
-void dirs_module_init(PyObject *mod);
-void manifest_module_init(PyObject *mod);
-void revlog_module_init(PyObject *mod);
-
-static const int version = 1;
-
-static void module_init(PyObject *mod)
-{
-	PyModule_AddIntConstant(mod, "version", version);
-
-	/* This module constant has two purposes.  First, it lets us unit test
-	 * the ImportError raised without hard-coding any error text.  This
-	 * means we can change the text in the future without breaking tests,
-	 * even across changesets without a recompile.  Second, its presence
-	 * can be used to determine whether the version-checking logic is
-	 * present, which also helps in testing across changesets without a
-	 * recompile.  Note that this means the pure-Python version of parsers
-	 * should not have this module constant. */
-	PyModule_AddStringConstant(mod, "versionerrortext", versionerrortext);
-
-	dirs_module_init(mod);
-	manifest_module_init(mod);
-	revlog_module_init(mod);
-
-	if (PyType_Ready(&dirstateTupleType) < 0)
-		return;
-	Py_INCREF(&dirstateTupleType);
-	PyModule_AddObject(mod, "dirstatetuple",
-			   (PyObject *)&dirstateTupleType);
-}
-
-static int check_python_version(void)
-{
-	PyObject *sys = PyImport_ImportModule("sys"), *ver;
-	long hexversion;
-	if (!sys)
-		return -1;
-	ver = PyObject_GetAttrString(sys, "hexversion");
-	Py_DECREF(sys);
-	if (!ver)
-		return -1;
-	hexversion = PyInt_AsLong(ver);
-	Py_DECREF(ver);
-	/* sys.hexversion is a 32-bit number by default, so the -1 case
-	 * should only occur in unusual circumstances (e.g. if sys.hexversion
-	 * is manually set to an invalid value). */
-	if ((hexversion == -1) || (hexversion >> 16 != PY_VERSION_HEX >> 16)) {
-		PyErr_Format(PyExc_ImportError, "%s: The Mercurial extension "
-			"modules were compiled with Python " PY_VERSION ", but "
-			"Mercurial is currently using Python with sys.hexversion=%ld: "
-			"Python %s\n at: %s", versionerrortext, hexversion,
-			Py_GetVersion(), Py_GetProgramFullPath());
-		return -1;
-	}
-	return 0;
-}
-
-#ifdef IS_PY3K
-static struct PyModuleDef parsers_module = {
-	PyModuleDef_HEAD_INIT,
-	"parsers",
-	parsers_doc,
-	-1,
-	methods
-};
-
-PyMODINIT_FUNC PyInit_parsers(void)
-{
-	PyObject *mod;
-
-	if (check_python_version() == -1)
-		return NULL;
-	mod = PyModule_Create(&parsers_module);
-	module_init(mod);
-	return mod;
-}
-#else
-PyMODINIT_FUNC initparsers(void)
-{
-	PyObject *mod;
-
-	if (check_python_version() == -1)
-		return;
-	mod = Py_InitModule3("parsers", methods, parsers_doc);
-	module_init(mod);
-}
-#endif

mercurial/cext/manifest.c

@@ -1,939 +1,939
 /*
  * manifest.c - manifest type that does on-demand parsing.
  *
  * Copyright 2015, Google Inc.
  *
  * This software may be used and distributed according to the terms of
  * the GNU General Public License, incorporated herein by reference.
  */
 #include <Python.h>
 
 #include <assert.h>
 #include <string.h>
 #include <stdlib.h>
 
 #include "util.h"
 
 #define DEFAULT_LINES 100000
 
 typedef struct {
 	char *start;
 	Py_ssize_t len; /* length of line including terminal newline */
 	char hash_suffix;
 	bool from_malloc;
 	bool deleted;
 } line;
 
 typedef struct {
 	PyObject_HEAD
 	PyObject *pydata;
 	line *lines;
 	int numlines; /* number of line entries */
 	int livelines; /* number of non-deleted lines */
 	int maxlines; /* allocated number of lines */
 	bool dirty;
 } lazymanifest;
 
 #define MANIFEST_OOM -1
 #define MANIFEST_NOT_SORTED -2
 #define MANIFEST_MALFORMED -3
 
-/* defined in parsers.c */
+/* defined in charencode.c */
 PyObject *unhexlify(const char *str, int len);
 
 /* get the length of the path for a line */
 static size_t pathlen(line *l) {
 	return strlen(l->start);
 }
 
 /* get the node value of a single line */
 static PyObject *nodeof(line *l) {
 	char *s = l->start;
 	ssize_t llen = pathlen(l);
 	PyObject *hash = unhexlify(s + llen + 1, 40);
 	if (!hash) {
 		return NULL;
 	}
 	if (l->hash_suffix != '\0') {
 		char newhash[21];
 		memcpy(newhash, PyBytes_AsString(hash), 20);
 		Py_DECREF(hash);
 		newhash[20] = l->hash_suffix;
 		hash = PyBytes_FromStringAndSize(newhash, 21);
 	}
 	return hash;
 }
 
 /* get the node hash and flags of a line as a tuple */
 static PyObject *hashflags(line *l)
 {
 	char *s = l->start;
 	size_t plen = pathlen(l);
 	PyObject *hash = nodeof(l);
 
 	/* 40 for hash, 1 for null byte, 1 for newline */
 	size_t hplen = plen + 42;
 	Py_ssize_t flen = l->len - hplen;
 	PyObject *flags;
 	PyObject *tup;
 
 	if (!hash)
 		return NULL;
 	flags = PyBytes_FromStringAndSize(s + hplen - 1, flen);
 	if (!flags) {
 		Py_DECREF(hash);
 		return NULL;
 	}
 	tup = PyTuple_Pack(2, hash, flags);
 	Py_DECREF(flags);
 	Py_DECREF(hash);
 	return tup;
 }
 
 /* if we're about to run out of space in the line index, add more */
 static bool realloc_if_full(lazymanifest *self)
 {
 	if (self->numlines == self->maxlines) {
 		self->maxlines *= 2;
 		self->lines = realloc(self->lines, self->maxlines * sizeof(line));
 	}
 	return !!self->lines;
 }
 
 /*
  * Find the line boundaries in the manifest that 'data' points to and store
  * information about each line in 'self'.
  */
 static int find_lines(lazymanifest *self, char *data, Py_ssize_t len)
 {
 	char *prev = NULL;
 	while (len > 0) {
 		line *l;
 		char *next = memchr(data, '\n', len);
 		if (!next) {
 			return MANIFEST_MALFORMED;
 		}
 		next++; /* advance past newline */
 		if (!realloc_if_full(self)) {
 			return MANIFEST_OOM; /* no memory */
 		}
 		if (prev && strcmp(prev, data) > -1) {
 			/* This data isn't sorted, so we have to abort. */
 			return MANIFEST_NOT_SORTED;
 		}
 		l = self->lines + ((self->numlines)++);
 		l->start = data;
 		l->len = next - data;
 		l->hash_suffix = '\0';
 		l->from_malloc = false;
 		l->deleted = false;
 		len = len - l->len;
 		prev = data;
 		data = next;
 	}
 	self->livelines = self->numlines;
 	return 0;
 }
 
 static int lazymanifest_init(lazymanifest *self, PyObject *args)
 {
 	char *data;
 	Py_ssize_t len;
 	int err, ret;
 	PyObject *pydata;
 	if (!PyArg_ParseTuple(args, "S", &pydata)) {
 		return -1;
 	}
 	err = PyBytes_AsStringAndSize(pydata, &data, &len);
 
 	self->dirty = false;
 	if (err == -1)
 		return -1;
 	self->pydata = pydata;
 	Py_INCREF(self->pydata);
 	Py_BEGIN_ALLOW_THREADS
 	self->lines = malloc(DEFAULT_LINES * sizeof(line));
 	self->maxlines = DEFAULT_LINES;
 	self->numlines = 0;
 	if (!self->lines)
 		ret = MANIFEST_OOM;
 	else
 		ret = find_lines(self, data, len);
 	Py_END_ALLOW_THREADS
 	switch (ret) {
 	case 0:
 		break;
 	case MANIFEST_OOM:
 		PyErr_NoMemory();
 		break;
 	case MANIFEST_NOT_SORTED:
 		PyErr_Format(PyExc_ValueError,
 			     "Manifest lines not in sorted order.");
 		break;
 	case MANIFEST_MALFORMED:
 		PyErr_Format(PyExc_ValueError,
 			     "Manifest did not end in a newline.");
 		break;
 	default:
 		PyErr_Format(PyExc_ValueError,
 			     "Unknown problem parsing manifest.");
 	}
 	return ret == 0 ? 0 : -1;
 }
 
 static void lazymanifest_dealloc(lazymanifest *self)
 {
 	/* free any extra lines we had to allocate */
 	int i;
 	for (i = 0; i < self->numlines; i++) {
 		if (self->lines[i].from_malloc) {
 			free(self->lines[i].start);
 		}
 	}
 	if (self->lines) {
 		free(self->lines);
 		self->lines = NULL;
 	}
 	if (self->pydata) {
 		Py_DECREF(self->pydata);
 		self->pydata = NULL;
 	}
 	PyObject_Del(self);
 }
 
 /* iteration support */
 
 typedef struct {
 	PyObject_HEAD lazymanifest *m;
 	Py_ssize_t pos;
 } lmIter;
 
 static void lmiter_dealloc(PyObject *o)
 {
 	lmIter *self = (lmIter *)o;
 	Py_DECREF(self->m);
 	PyObject_Del(self);
 }
 
 static line *lmiter_nextline(lmIter *self)
 {
 	do {
 		self->pos++;
 		if (self->pos >= self->m->numlines) {
 			return NULL;
 		}
 		/* skip over deleted manifest entries */
 	} while (self->m->lines[self->pos].deleted);
 	return self->m->lines + self->pos;
 }
 
 static PyObject *lmiter_iterentriesnext(PyObject *o)
 {
 	size_t pl;
 	line *l;
 	Py_ssize_t consumed;
 	PyObject *ret = NULL, *path = NULL, *hash = NULL, *flags = NULL;
 	l = lmiter_nextline((lmIter *)o);
 	if (!l) {
 		goto done;
 	}
 	pl = pathlen(l);
 	path = PyBytes_FromStringAndSize(l->start, pl);
 	hash = nodeof(l);
 	consumed = pl + 41;
 	flags = PyBytes_FromStringAndSize(l->start + consumed,
 					   l->len - consumed - 1);
 	if (!path || !hash || !flags) {
 		goto done;
 	}
 	ret = PyTuple_Pack(3, path, hash, flags);
 done:
 	Py_XDECREF(path);
 	Py_XDECREF(hash);
 	Py_XDECREF(flags);
 	return ret;
 }
 
 #ifdef IS_PY3K
 #define LAZYMANIFESTENTRIESITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT
 #else
 #define LAZYMANIFESTENTRIESITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT \
 	| Py_TPFLAGS_HAVE_ITER
 #endif
 
 static PyTypeObject lazymanifestEntriesIterator = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"parsers.lazymanifest.entriesiterator", /*tp_name */
 	sizeof(lmIter),                  /*tp_basicsize */
 	0,                               /*tp_itemsize */
 	lmiter_dealloc,                  /*tp_dealloc */
 	0,                               /*tp_print */
 	0,                               /*tp_getattr */
 	0,                               /*tp_setattr */
 	0,                               /*tp_compare */
 	0,                               /*tp_repr */
 	0,                               /*tp_as_number */
 	0,                               /*tp_as_sequence */
 	0,                               /*tp_as_mapping */
 	0,                               /*tp_hash */
 	0,                               /*tp_call */
 	0,                               /*tp_str */
 	0,                               /*tp_getattro */
 	0,                               /*tp_setattro */
 	0,                               /*tp_as_buffer */
 	LAZYMANIFESTENTRIESITERATOR_TPFLAGS, /* tp_flags */
 	"Iterator for 3-tuples in a lazymanifest.",  /* tp_doc */
 	0,                               /* tp_traverse */
 	0,                               /* tp_clear */
 	0,                               /* tp_richcompare */
 	0,                               /* tp_weaklistoffset */
 	PyObject_SelfIter,               /* tp_iter: __iter__() method */
 	lmiter_iterentriesnext,          /* tp_iternext: next() method */
 };
 
 static PyObject *lmiter_iterkeysnext(PyObject *o)
 {
 	size_t pl;
 	line *l = lmiter_nextline((lmIter *)o);
 	if (!l) {
 		return NULL;
 	}
 	pl = pathlen(l);
 	return PyBytes_FromStringAndSize(l->start, pl);
 }
 
 #ifdef IS_PY3K
 #define LAZYMANIFESTKEYSITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT
 #else
 #define LAZYMANIFESTKEYSITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT \
 	| Py_TPFLAGS_HAVE_ITER
 #endif
 
 static PyTypeObject lazymanifestKeysIterator = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"parsers.lazymanifest.keysiterator", /*tp_name */
 	sizeof(lmIter),                  /*tp_basicsize */
 	0,                               /*tp_itemsize */
 	lmiter_dealloc,                  /*tp_dealloc */
 	0,                               /*tp_print */
 	0,                               /*tp_getattr */
 	0,                               /*tp_setattr */
 	0,                               /*tp_compare */
 	0,                               /*tp_repr */
 	0,                               /*tp_as_number */
 	0,                               /*tp_as_sequence */
 	0,                               /*tp_as_mapping */
 	0,                               /*tp_hash */
 	0,                               /*tp_call */
 	0,                               /*tp_str */
 	0,                               /*tp_getattro */
 	0,                               /*tp_setattro */
 	0,                               /*tp_as_buffer */
 	LAZYMANIFESTKEYSITERATOR_TPFLAGS, /* tp_flags */
 	"Keys iterator for a lazymanifest.",  /* tp_doc */
 	0,                               /* tp_traverse */
 	0,                               /* tp_clear */
 	0,                               /* tp_richcompare */
 	0,                               /* tp_weaklistoffset */
 	PyObject_SelfIter,               /* tp_iter: __iter__() method */
 	lmiter_iterkeysnext,             /* tp_iternext: next() method */
 };
 
 static lazymanifest *lazymanifest_copy(lazymanifest *self);
 
 static PyObject *lazymanifest_getentriesiter(lazymanifest *self)
 {
 	lmIter *i = NULL;
 	lazymanifest *t = lazymanifest_copy(self);
 	if (!t) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 	i = PyObject_New(lmIter, &lazymanifestEntriesIterator);
 	if (i) {
 		i->m = t;
 		i->pos = -1;
 	} else {
 		Py_DECREF(t);
 		PyErr_NoMemory();
 	}
 	return (PyObject *)i;
 }
 
 static PyObject *lazymanifest_getkeysiter(lazymanifest *self)
 {
 	lmIter *i = NULL;
 	lazymanifest *t = lazymanifest_copy(self);
 	if (!t) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 	i = PyObject_New(lmIter, &lazymanifestKeysIterator);
 	if (i) {
 		i->m = t;
 		i->pos = -1;
 	} else {
 		Py_DECREF(t);
 		PyErr_NoMemory();
 	}
 	return (PyObject *)i;
 }
 
 /* __getitem__ and __setitem__ support */
 
 static Py_ssize_t lazymanifest_size(lazymanifest *self)
 {
 	return self->livelines;
 }
 
 static int linecmp(const void *left, const void *right)
 {
 	return strcmp(((const line *)left)->start,
 		      ((const line *)right)->start);
 }
 
 static PyObject *lazymanifest_getitem(lazymanifest *self, PyObject *key)
 {
 	line needle;
 	line *hit;
 	if (!PyBytes_Check(key)) {
 		PyErr_Format(PyExc_TypeError,
 			     "getitem: manifest keys must be a string.");
 		return NULL;
 	}
 	needle.start = PyBytes_AsString(key);
 	hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
 		      &linecmp);
 	if (!hit || hit->deleted) {
 		PyErr_Format(PyExc_KeyError, "No such manifest entry.");
 		return NULL;
 	}
 	return hashflags(hit);
 }
 
 static int lazymanifest_delitem(lazymanifest *self, PyObject *key)
 {
 	line needle;
 	line *hit;
 	if (!PyBytes_Check(key)) {
 		PyErr_Format(PyExc_TypeError,
 			     "delitem: manifest keys must be a string.");
 		return -1;
 	}
 	needle.start = PyBytes_AsString(key);
 	hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
 		      &linecmp);
 	if (!hit || hit->deleted) {
 		PyErr_Format(PyExc_KeyError,
 			     "Tried to delete nonexistent manifest entry.");
 		return -1;
 	}
 	self->dirty = true;
 	hit->deleted = true;
 	self->livelines--;
 	return 0;
 }
 
 /* Do a binary search for the insertion point for new, creating the
  * new entry if needed. */
 static int internalsetitem(lazymanifest *self, line *new) {
 	int start = 0, end = self->numlines;
 	while (start < end) {
 		int pos = start + (end - start) / 2;
 		int c = linecmp(new, self->lines + pos);
 		if (c < 0)
 			end = pos;
 		else if (c > 0)
 			start = pos + 1;
 		else {
 			if (self->lines[pos].deleted)
 				self->livelines++;
 			if (self->lines[pos].from_malloc)
 				free(self->lines[pos].start);
 			start = pos;
 			goto finish;
 		}
 	}
 	/* being here means we need to do an insert */
 	if (!realloc_if_full(self)) {
 		PyErr_NoMemory();
 		return -1;
 	}
 	memmove(self->lines + start + 1, self->lines + start,
 		(self->numlines - start) * sizeof(line));
 	self->numlines++;
 	self->livelines++;
 finish:
 	self->lines[start] = *new;
 	self->dirty = true;
 	return 0;
 }
 
 static int lazymanifest_setitem(
 	lazymanifest *self, PyObject *key, PyObject *value)
 {
 	char *path;
 	Py_ssize_t plen;
 	PyObject *pyhash;
 	Py_ssize_t hlen;
 	char *hash;
 	PyObject *pyflags;
 	char *flags;
 	Py_ssize_t flen;
 	size_t dlen;
 	char *dest;
 	int i;
 	line new;
 	if (!PyBytes_Check(key)) {
 		PyErr_Format(PyExc_TypeError,
 			     "setitem: manifest keys must be a string.");
 		return -1;
 	}
 	if (!value) {
 		return lazymanifest_delitem(self, key);
 	}
 	if (!PyTuple_Check(value) || PyTuple_Size(value) != 2) {
 		PyErr_Format(PyExc_TypeError,
 			     "Manifest values must be a tuple of (node, flags).");
 		return -1;
 	}
 	if (PyBytes_AsStringAndSize(key, &path, &plen) == -1) {
 		return -1;
 	}
 
 	pyhash = PyTuple_GetItem(value, 0);
 	if (!PyBytes_Check(pyhash)) {
 		PyErr_Format(PyExc_TypeError,
 			     "node must be a 20-byte string");
 		return -1;
 	}
 	hlen = PyBytes_Size(pyhash);
 	/* Some parts of the codebase try and set 21 or 22
 	 * byte "hash" values in order to perturb things for
 	 * status. We have to preserve at least the 21st
 	 * byte. Sigh. If there's a 22nd byte, we drop it on
 	 * the floor, which works fine.
 	 */
 	if (hlen != 20 && hlen != 21 && hlen != 22) {
 		PyErr_Format(PyExc_TypeError,
 			     "node must be a 20-byte string");
 		return -1;
 	}
 	hash = PyBytes_AsString(pyhash);
 
 	pyflags = PyTuple_GetItem(value, 1);
 	if (!PyBytes_Check(pyflags) || PyBytes_Size(pyflags) > 1) {
 		PyErr_Format(PyExc_TypeError,
 			     "flags must a 0 or 1 byte string");
 		return -1;
 	}
 	if (PyBytes_AsStringAndSize(pyflags, &flags, &flen) == -1) {
 		return -1;
 	}
 	/* one null byte and one newline */
 	dlen = plen + 41 + flen + 1;
 	dest = malloc(dlen);
 	if (!dest) {
 		PyErr_NoMemory();
 		return -1;
 	}
 	memcpy(dest, path, plen + 1);
 	for (i = 0; i < 20; i++) {
 		/* Cast to unsigned, so it will not get sign-extended when promoted
 		 * to int (as is done when passing to a variadic function)
 		 */
 		sprintf(dest + plen + 1 + (i * 2), "%02x", (unsigned char)hash[i]);
 	}
 	memcpy(dest + plen + 41, flags, flen);
 	dest[plen + 41 + flen] = '\n';
 	new.start = dest;
 	new.len = dlen;
 	new.hash_suffix = '\0';
 	if (hlen > 20) {
 		new.hash_suffix = hash[20];
 	}
 	new.from_malloc = true;     /* is `start` a pointer we allocated? */
 	new.deleted = false;        /* is this entry deleted? */
 	if (internalsetitem(self, &new)) {
 		return -1;
 	}
 	return 0;
 }
 
 static PyMappingMethods lazymanifest_mapping_methods = {
 	(lenfunc)lazymanifest_size,             /* mp_length */
 	(binaryfunc)lazymanifest_getitem,       /* mp_subscript */
 	(objobjargproc)lazymanifest_setitem,    /* mp_ass_subscript */
 };
 
 /* sequence methods (important or __contains__ builds an iterator) */
 
 static int lazymanifest_contains(lazymanifest *self, PyObject *key)
 {
 	line needle;
 	line *hit;
 	if (!PyBytes_Check(key)) {
 		/* Our keys are always strings, so if the contains
 		 * check is for a non-string, just return false. */
 		return 0;
 	}
 	needle.start = PyBytes_AsString(key);
 	hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
 		      &linecmp);
 	if (!hit || hit->deleted) {
 		return 0;
 	}
 	return 1;
 }
 
 static PySequenceMethods lazymanifest_seq_meths = {
 	(lenfunc)lazymanifest_size, /* sq_length */
 	0, /* sq_concat */
 	0, /* sq_repeat */
 	0, /* sq_item */
 	0, /* sq_slice */
 	0, /* sq_ass_item */
 	0, /* sq_ass_slice */
 	(objobjproc)lazymanifest_contains, /* sq_contains */
 	0, /* sq_inplace_concat */
 	0, /* sq_inplace_repeat */
 };
 
 
 /* Other methods (copy, diff, etc) */
 static PyTypeObject lazymanifestType;
 
 /* If the manifest has changes, build the new manifest text and reindex it. */
 static int compact(lazymanifest *self) {
 	int i;
 	ssize_t need = 0;
 	char *data;
 	line *src, *dst;
 	PyObject *pydata;
 	if (!self->dirty)
 		return 0;
 	for (i = 0; i < self->numlines; i++) {
 		if (!self->lines[i].deleted) {
 			need += self->lines[i].len;
 		}
 	}
 	pydata = PyBytes_FromStringAndSize(NULL, need);
 	if (!pydata)
 		return -1;
 	data = PyBytes_AsString(pydata);
 	if (!data) {
 		return -1;
 	}
 	src = self->lines;
 	dst = self->lines;
 	for (i = 0; i < self->numlines; i++, src++) {
 		char *tofree = NULL;
 		if (src->from_malloc) {
 			tofree = src->start;
 		}
 		if (!src->deleted) {
 			memcpy(data, src->start, src->len);
 			*dst = *src;
 			dst->start = data;
 			dst->from_malloc = false;
 			data += dst->len;
 			dst++;
 		}
 		free(tofree);
 	}
 	Py_DECREF(self->pydata);
 	self->pydata = pydata;
 	self->numlines = self->livelines;
 	self->dirty = false;
 	return 0;
 }
 
 static PyObject *lazymanifest_text(lazymanifest *self)
 {
 	if (compact(self) != 0) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 	Py_INCREF(self->pydata);
 	return self->pydata;
 }
 
 static lazymanifest *lazymanifest_copy(lazymanifest *self)
 {
 	lazymanifest *copy = NULL;
 	if (compact(self) != 0) {
 		goto nomem;
 	}
 	copy = PyObject_New(lazymanifest, &lazymanifestType);
 	if (!copy) {
 		goto nomem;
 	}
 	copy->numlines = self->numlines;
 	copy->livelines = self->livelines;
 	copy->dirty = false;
 	copy->lines = malloc(self->maxlines *sizeof(line));
 	if (!copy->lines) {
 		goto nomem;
 	}
 	memcpy(copy->lines, self->lines, self->numlines * sizeof(line));
 	copy->maxlines = self->maxlines;
 	copy->pydata = self->pydata;
 	Py_INCREF(copy->pydata);
 	return copy;
 nomem:
 	PyErr_NoMemory();
 	Py_XDECREF(copy);
 	return NULL;
 }
 
 static lazymanifest *lazymanifest_filtercopy(
 	lazymanifest *self, PyObject *matchfn)
 {
 	lazymanifest *copy = NULL;
 	int i;
 	if (!PyCallable_Check(matchfn)) {
 		PyErr_SetString(PyExc_TypeError, "matchfn must be callable");
 		return NULL;
 	}
 	/* compact ourselves first to avoid double-frees later when we
 	 * compact tmp so that it doesn't have random pointers to our
 	 * underlying from_malloc-data (self->pydata is safe) */
 	if (compact(self) != 0) {
 		goto nomem;
 	}
 	copy = PyObject_New(lazymanifest, &lazymanifestType);
 	if (!copy) {
 		goto nomem;
 	}
 	copy->dirty = true;
 	copy->lines = malloc(self->maxlines * sizeof(line));
 	if (!copy->lines) {
 		goto nomem;
 	}
 	copy->maxlines = self->maxlines;
 	copy->numlines = 0;
 	copy->pydata = self->pydata;
 	Py_INCREF(self->pydata);
 	for (i = 0; i < self->numlines; i++) {
 		PyObject *arglist = NULL, *result = NULL;
 		arglist = Py_BuildValue("(s)", self->lines[i].start);
 		if (!arglist) {
 			return NULL;
 		}
 		result = PyObject_CallObject(matchfn, arglist);
 		Py_DECREF(arglist);
 		/* if the callback raised an exception, just let it
 		 * through and give up */
 		if (!result) {
 			free(copy->lines);
 			Py_DECREF(self->pydata);
 			return NULL;
 		}
 		if (PyObject_IsTrue(result)) {
 			assert(!(self->lines[i].from_malloc));
 			copy->lines[copy->numlines++] = self->lines[i];
 		}
 		Py_DECREF(result);
 	}
 	copy->livelines = copy->numlines;
 	return copy;
 nomem:
 	PyErr_NoMemory();
 	Py_XDECREF(copy);
 	return NULL;
 }
 
 static PyObject *lazymanifest_diff(lazymanifest *self, PyObject *args)
 {
 	lazymanifest *other;
 	PyObject *pyclean = NULL;
 	bool listclean;
 	PyObject *emptyTup = NULL, *ret = NULL;
 	PyObject *es;
 	int sneedle = 0, oneedle = 0;
 	if (!PyArg_ParseTuple(args, "O!|O", &lazymanifestType, &other, &pyclean)) {
 		return NULL;
 	}
 	listclean = (!pyclean) ? false : PyObject_IsTrue(pyclean);
 	es = PyBytes_FromString("");
 	if (!es) {
 		goto nomem;
 	}
 	emptyTup = PyTuple_Pack(2, Py_None, es);
 	Py_DECREF(es);
 	if (!emptyTup) {
 		goto nomem;
 	}
 	ret = PyDict_New();
 	if (!ret) {
 		goto nomem;
 	}
 	while (sneedle != self->numlines || oneedle != other->numlines) {
 		line *left = self->lines + sneedle;
 		line *right = other->lines + oneedle;
 		int result;
 		PyObject *key;
 		PyObject *outer;
 		/* If we're looking at a deleted entry and it's not
 		 * the end of the manifest, just skip it. */
 		if (left->deleted && sneedle < self->numlines) {
 			sneedle++;
 			continue;
 		}
 		if (right->deleted && oneedle < other->numlines) {
 			oneedle++;
 			continue;
 		}
 		/* if we're at the end of either manifest, then we
 		 * know the remaining items are adds so we can skip
 		 * the strcmp. */
 		if (sneedle == self->numlines) {
 			result = 1;
 		} else if (oneedle == other->numlines) {
 			result = -1;
 		} else {
 			result = linecmp(left, right);
 		}
 		key = result <= 0 ?
 			PyBytes_FromString(left->start) :
 			PyBytes_FromString(right->start);
 		if (!key)
 			goto nomem;
 		if (result < 0) {
 			PyObject *l = hashflags(left);
 			if (!l) {
 				goto nomem;
 			}
 			outer = PyTuple_Pack(2, l, emptyTup);
 			Py_DECREF(l);
 			if (!outer) {
 				goto nomem;
 			}
 			PyDict_SetItem(ret, key, outer);
 			Py_DECREF(outer);
 			sneedle++;
 		} else if (result > 0) {
 			PyObject *r = hashflags(right);
 			if (!r) {
 				goto nomem;
 			}
 			outer = PyTuple_Pack(2, emptyTup, r);
 			Py_DECREF(r);
 			if (!outer) {
 				goto nomem;
 			}
 			PyDict_SetItem(ret, key, outer);
 			Py_DECREF(outer);
 			oneedle++;
 		} else {
 			/* file exists in both manifests */
 			if (left->len != right->len
 			    || memcmp(left->start, right->start, left->len)
 			    || left->hash_suffix != right->hash_suffix) {
 				PyObject *l = hashflags(left);
 				PyObject *r;
 				if (!l) {
 					goto nomem;
 				}
 				r = hashflags(right);
 				if (!r) {
 					Py_DECREF(l);
 					goto nomem;
 				}
 				outer = PyTuple_Pack(2, l, r);
 				Py_DECREF(l);
 				Py_DECREF(r);
 				if (!outer) {
 					goto nomem;
 				}
 				PyDict_SetItem(ret, key, outer);
 				Py_DECREF(outer);
 			} else if (listclean) {
 				PyDict_SetItem(ret, key, Py_None);
 			}
 			sneedle++;
 			oneedle++;
 		}
 		Py_DECREF(key);
 	}
 	Py_DECREF(emptyTup);
 	return ret;
 nomem:
 	PyErr_NoMemory();
 	Py_XDECREF(ret);
 	Py_XDECREF(emptyTup);
 	return NULL;
 }
 
 static PyMethodDef lazymanifest_methods[] = {
 	{"iterkeys", (PyCFunction)lazymanifest_getkeysiter, METH_NOARGS,
 	 "Iterate over file names in this lazymanifest."},
 	{"iterentries", (PyCFunction)lazymanifest_getentriesiter, METH_NOARGS,
 	 "Iterate over (path, nodeid, flags) tuples in this lazymanifest."},
 	{"copy", (PyCFunction)lazymanifest_copy, METH_NOARGS,
 	 "Make a copy of this lazymanifest."},
 	{"filtercopy", (PyCFunction)lazymanifest_filtercopy, METH_O,
 	 "Make a copy of this manifest filtered by matchfn."},
 	{"diff", (PyCFunction)lazymanifest_diff, METH_VARARGS,
 	 "Compare this lazymanifest to another one."},
 	{"text", (PyCFunction)lazymanifest_text, METH_NOARGS,
 	 "Encode this manifest to text."},
 	{NULL},
 };
 
 #ifdef IS_PY3K
 #define LAZYMANIFEST_TPFLAGS Py_TPFLAGS_DEFAULT
 #else
 #define LAZYMANIFEST_TPFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_SEQUENCE_IN
 #endif
 
 static PyTypeObject lazymanifestType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"parsers.lazymanifest",                           /* tp_name */
 	sizeof(lazymanifest),                             /* tp_basicsize */
 	0,                                                /* tp_itemsize */
 	(destructor)lazymanifest_dealloc,                 /* tp_dealloc */
 	0,                                                /* tp_print */
 	0,                                                /* tp_getattr */
 	0,                                                /* tp_setattr */
 	0,                                                /* tp_compare */
 	0,                                                /* tp_repr */
 	0,                                                /* tp_as_number */
 	&lazymanifest_seq_meths,                          /* tp_as_sequence */
 	&lazymanifest_mapping_methods,                    /* tp_as_mapping */
 	0,                                                /* tp_hash */
 	0,                                                /* tp_call */
 	0,                                                /* tp_str */
 	0,                                                /* tp_getattro */
 	0,                                                /* tp_setattro */
 	0,                                                /* tp_as_buffer */
 	LAZYMANIFEST_TPFLAGS,                             /* tp_flags */
 	"TODO(augie)",                                    /* tp_doc */
 	0,                                                /* tp_traverse */
 	0,                                                /* tp_clear */
 	0,                                                /* tp_richcompare */
 	0,                                             /* tp_weaklistoffset */
 	(getiterfunc)lazymanifest_getkeysiter,                /* tp_iter */
 	0,                                                /* tp_iternext */
 	lazymanifest_methods,                             /* tp_methods */
 	0,                                                /* tp_members */
 	0,                                                /* tp_getset */
 	0,                                                /* tp_base */
 	0,                                                /* tp_dict */
 	0,                                                /* tp_descr_get */
 	0,                                                /* tp_descr_set */
 	0,                                                /* tp_dictoffset */
 	(initproc)lazymanifest_init,                      /* tp_init */
 	0,                                                /* tp_alloc */
 };
 
 void manifest_module_init(PyObject * mod)
 {
 	lazymanifestType.tp_new = PyType_GenericNew;
 	if (PyType_Ready(&lazymanifestType) < 0)
 		return;
 	Py_INCREF(&lazymanifestType);
 
 	PyModule_AddObject(mod, "lazymanifest",
 			   (PyObject *)&lazymanifestType);
 }

mercurial/cext/parsers.c

@@ -1,997 +1,806
 /*
  parsers.c - efficient content parsing
 
  Copyright 2008 Matt Mackall <[e-mail]> and others
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 */
 
 #include <Python.h>
 #include <ctype.h>
 #include <stddef.h>
 #include <string.h>
 
 #include "util.h"
 #include "bitmanipulation.h"
 
 #ifdef IS_PY3K
 /* The mapping of Python types is meant to be temporary to get Python
  * 3 to compile. We should remove this once Python 3 support is fully
  * supported and proper types are used in the extensions themselves. */
 #define PyInt_Type PyLong_Type
 #define PyInt_Check PyLong_Check
 #define PyInt_FromLong PyLong_FromLong
 #define PyInt_FromSsize_t PyLong_FromSsize_t
 #define PyInt_AS_LONG PyLong_AS_LONG
 #define PyInt_AsLong PyLong_AsLong
 #endif
 
 static const char *const versionerrortext = "Python minor version mismatch";
 
-static const char lowertable[128] = {
-	'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07',
-	'\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
-	'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17',
-	'\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
-	'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27',
-	'\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
-	'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37',
-	'\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
-	'\x40',
-	        '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67', /* A-G */
-	'\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f', /* H-O */
-	'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77', /* P-W */
-	'\x78', '\x79', '\x7a',                                         /* X-Z */
-	                        '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
-	'\x60', '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67',
-	'\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f',
-	'\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77',
-	'\x78', '\x79', '\x7a', '\x7b', '\x7c', '\x7d', '\x7e', '\x7f'
-};
-
-static const char uppertable[128] = {
-	'\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07',
-	'\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f',
-	'\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17',
-	'\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f',
-	'\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27',
-	'\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f',
-	'\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37',
-	'\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f',
-	'\x40', '\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47',
-	'\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f',
-	'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57',
-	'\x58', '\x59', '\x5a', '\x5b', '\x5c', '\x5d', '\x5e', '\x5f',
-	'\x60',
-		'\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47', /* a-g */
-	'\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f', /* h-o */
-	'\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57', /* p-w */
-	'\x58', '\x59', '\x5a', 					/* x-z */
-				'\x7b', '\x7c', '\x7d', '\x7e', '\x7f'
-};
-
-/*
- * Turn a hex-encoded string into binary.
- */
-PyObject *unhexlify(const char *str, int len)
-{
-	PyObject *ret;
-	char *d;
-	int i;
-
-	ret = PyBytes_FromStringAndSize(NULL, len / 2);
-
-	if (!ret)
-		return NULL;
-
-	d = PyBytes_AsString(ret);
-
-	for (i = 0; i < len;) {
-		int hi = hexdigit(str, i++);
-		int lo = hexdigit(str, i++);
-		*d++ = (hi << 4) | lo;
-	}
-
-	return ret;
-}
-
-static inline PyObject *_asciitransform(PyObject *str_obj,
-					const char table[128],
-					PyObject *fallback_fn)
-{
-	char *str, *newstr;
-	Py_ssize_t i, len;
-	PyObject *newobj = NULL;
-	PyObject *ret = NULL;
-
-	str = PyBytes_AS_STRING(str_obj);
-	len = PyBytes_GET_SIZE(str_obj);
-
-	newobj = PyBytes_FromStringAndSize(NULL, len);
-	if (!newobj)
-		goto quit;
-
-	newstr = PyBytes_AS_STRING(newobj);
-
-	for (i = 0; i < len; i++) {
-		char c = str[i];
-		if (c & 0x80) {
-			if (fallback_fn != NULL) {
-				ret = PyObject_CallFunctionObjArgs(fallback_fn,
-					str_obj, NULL);
-			} else {
-				PyObject *err = PyUnicodeDecodeError_Create(
-					"ascii", str, len, i, (i + 1),
-					"unexpected code byte");
-				PyErr_SetObject(PyExc_UnicodeDecodeError, err);
-				Py_XDECREF(err);
-			}
-			goto quit;
-		}
-		newstr[i] = table[(unsigned char)c];
-	}
-
-	ret = newobj;
-	Py_INCREF(ret);
-quit:
-	Py_XDECREF(newobj);
-	return ret;
-}
-
-static PyObject *asciilower(PyObject *self, PyObject *args)
-{
-	PyObject *str_obj;
-	if (!PyArg_ParseTuple(args, "O!:asciilower", &PyBytes_Type, &str_obj))
-		return NULL;
-	return _asciitransform(str_obj, lowertable, NULL);
-}
-
-static PyObject *asciiupper(PyObject *self, PyObject *args)
-{
-	PyObject *str_obj;
-	if (!PyArg_ParseTuple(args, "O!:asciiupper", &PyBytes_Type, &str_obj))
-		return NULL;
-	return _asciitransform(str_obj, uppertable, NULL);
-}
+/* defined in charencode.c */
+PyObject *unhexlify(const char *str, int len);
+PyObject *asciilower(PyObject *self, PyObject *args);
+PyObject *asciiupper(PyObject *self, PyObject *args);
+PyObject *make_file_foldmap(PyObject *self, PyObject *args);
 
 static PyObject *dict_new_presized(PyObject *self, PyObject *args)
 {
 	Py_ssize_t expected_size;
 
 	if (!PyArg_ParseTuple(args, "n:make_presized_dict", &expected_size))
 		return NULL;
 
 	return _dict_new_presized(expected_size);
 }
 
-static PyObject *make_file_foldmap(PyObject *self, PyObject *args)
-{
-	PyObject *dmap, *spec_obj, *normcase_fallback;
-	PyObject *file_foldmap = NULL;
-	enum normcase_spec spec;
-	PyObject *k, *v;
-	dirstateTupleObject *tuple;
-	Py_ssize_t pos = 0;
-	const char *table;
-
-	if (!PyArg_ParseTuple(args, "O!O!O!:make_file_foldmap",
-			      &PyDict_Type, &dmap,
-			      &PyInt_Type, &spec_obj,
-			      &PyFunction_Type, &normcase_fallback))
-		goto quit;
-
-	spec = (int)PyInt_AS_LONG(spec_obj);
-	switch (spec) {
-	case NORMCASE_LOWER:
-		table = lowertable;
-		break;
-	case NORMCASE_UPPER:
-		table = uppertable;
-		break;
-	case NORMCASE_OTHER:
-		table = NULL;
-		break;
-	default:
-		PyErr_SetString(PyExc_TypeError, "invalid normcasespec");
-		goto quit;
-	}
-
-	/* Add some more entries to deal with additions outside this
-	   function. */
-	file_foldmap = _dict_new_presized((PyDict_Size(dmap) / 10) * 11);
-	if (file_foldmap == NULL)
-		goto quit;
-
-	while (PyDict_Next(dmap, &pos, &k, &v)) {
-		if (!dirstate_tuple_check(v)) {
-			PyErr_SetString(PyExc_TypeError,
-					"expected a dirstate tuple");
-			goto quit;
-		}
-
-		tuple = (dirstateTupleObject *)v;
-		if (tuple->state != 'r') {
-			PyObject *normed;
-			if (table != NULL) {
-				normed = _asciitransform(k, table,
-					normcase_fallback);
-			} else {
-				normed = PyObject_CallFunctionObjArgs(
-					normcase_fallback, k, NULL);
-			}
-
-			if (normed == NULL)
-				goto quit;
-			if (PyDict_SetItem(file_foldmap, normed, k) == -1) {
-				Py_DECREF(normed);
-				goto quit;
-			}
-			Py_DECREF(normed);
-		}
-	}
-	return file_foldmap;
-quit:
-	Py_XDECREF(file_foldmap);
-	return NULL;
-}
-
 /*
  * This code assumes that a manifest is stitched together with newline
  * ('\n') characters.
  */
 static PyObject *parse_manifest(PyObject *self, PyObject *args)
 {
 	PyObject *mfdict, *fdict;
 	char *str, *start, *end;
 	int len;
 
 	if (!PyArg_ParseTuple(args, "O!O!s#:parse_manifest",
 			      &PyDict_Type, &mfdict,
 			      &PyDict_Type, &fdict,
 			      &str, &len))
 		goto quit;
 
 	start = str;
 	end = str + len;
 	while (start < end) {
 		PyObject *file = NULL, *node = NULL;
 		PyObject *flags = NULL;
 		char *zero = NULL, *newline = NULL;
 		ptrdiff_t nlen;
 
 		zero = memchr(start, '\0', end - start);
 		if (!zero) {
 			PyErr_SetString(PyExc_ValueError,
 					"manifest entry has no separator");
 			goto quit;
 		}
 
 		newline = memchr(zero + 1, '\n', end - (zero + 1));
 		if (!newline) {
 			PyErr_SetString(PyExc_ValueError,
 					"manifest contains trailing garbage");
 			goto quit;
 		}
 
 		file = PyBytes_FromStringAndSize(start, zero - start);
 
 		if (!file)
 			goto bail;
 
 		nlen = newline - zero - 1;
 
 		node = unhexlify(zero + 1, nlen > 40 ? 40 : (int)nlen);
 		if (!node)
 			goto bail;
 
 		if (nlen > 40) {
 			flags = PyBytes_FromStringAndSize(zero + 41,
 							   nlen - 40);
 			if (!flags)
 				goto bail;
 
 			if (PyDict_SetItem(fdict, file, flags) == -1)
 				goto bail;
 		}
 
 		if (PyDict_SetItem(mfdict, file, node) == -1)
 			goto bail;
 
 		start = newline + 1;
 
 		Py_XDECREF(flags);
 		Py_XDECREF(node);
 		Py_XDECREF(file);
 		continue;
 	bail:
 		Py_XDECREF(flags);
 		Py_XDECREF(node);
 		Py_XDECREF(file);
 		goto quit;
 	}
 
 	Py_INCREF(Py_None);
 	return Py_None;
 quit:
 	return NULL;
 }
 
 static inline dirstateTupleObject *make_dirstate_tuple(char state, int mode,
 						       int size, int mtime)
 {
 	dirstateTupleObject *t = PyObject_New(dirstateTupleObject,
 					      &dirstateTupleType);
 	if (!t)
 		return NULL;
 	t->state = state;
 	t->mode = mode;
 	t->size = size;
 	t->mtime = mtime;
 	return t;
 }
 
 static PyObject *dirstate_tuple_new(PyTypeObject *subtype, PyObject *args,
 				    PyObject *kwds)
 {
 	/* We do all the initialization here and not a tp_init function because
 	 * dirstate_tuple is immutable. */
 	dirstateTupleObject *t;
 	char state;
 	int size, mode, mtime;
 	if (!PyArg_ParseTuple(args, "ciii", &state, &mode, &size, &mtime))
 		return NULL;
 
 	t = (dirstateTupleObject *)subtype->tp_alloc(subtype, 1);
 	if (!t)
 		return NULL;
 	t->state = state;
 	t->mode = mode;
 	t->size = size;
 	t->mtime = mtime;
 
 	return (PyObject *)t;
 }
 
 static void dirstate_tuple_dealloc(PyObject *o)
 {
 	PyObject_Del(o);
 }
 
 static Py_ssize_t dirstate_tuple_length(PyObject *o)
 {
 	return 4;
 }
 
 static PyObject *dirstate_tuple_item(PyObject *o, Py_ssize_t i)
 {
 	dirstateTupleObject *t = (dirstateTupleObject *)o;
 	switch (i) {
 	case 0:
 		return PyBytes_FromStringAndSize(&t->state, 1);
 	case 1:
 		return PyInt_FromLong(t->mode);
 	case 2:
 		return PyInt_FromLong(t->size);
 	case 3:
 		return PyInt_FromLong(t->mtime);
 	default:
 		PyErr_SetString(PyExc_IndexError, "index out of range");
 		return NULL;
 	}
 }
 
 static PySequenceMethods dirstate_tuple_sq = {
 	dirstate_tuple_length,     /* sq_length */
 	0,                         /* sq_concat */
 	0,                         /* sq_repeat */
 	dirstate_tuple_item,       /* sq_item */
 	0,                         /* sq_ass_item */
 	0,                         /* sq_contains */
 	0,                         /* sq_inplace_concat */
 	0                          /* sq_inplace_repeat */
 };
 
 PyTypeObject dirstateTupleType = {
 	PyVarObject_HEAD_INIT(NULL, 0)
 	"dirstate_tuple",          /* tp_name */
 	sizeof(dirstateTupleObject),/* tp_basicsize */
 	0,                         /* tp_itemsize */
 	(destructor)dirstate_tuple_dealloc, /* tp_dealloc */
 	0,                         /* tp_print */
 	0,                         /* tp_getattr */
 	0,                         /* tp_setattr */
 	0,                         /* tp_compare */
 	0,                         /* tp_repr */
 	0,                         /* tp_as_number */
 	&dirstate_tuple_sq,        /* tp_as_sequence */
 	0,                         /* tp_as_mapping */
 	0,                         /* tp_hash  */
 	0,                         /* tp_call */
 	0,                         /* tp_str */
 	0,                         /* tp_getattro */
 	0,                         /* tp_setattro */
 	0,                         /* tp_as_buffer */
 	Py_TPFLAGS_DEFAULT,        /* tp_flags */
 	"dirstate tuple",          /* tp_doc */
 	0,                         /* tp_traverse */
 	0,                         /* tp_clear */
 	0,                         /* tp_richcompare */
 	0,                         /* tp_weaklistoffset */
 	0,                         /* tp_iter */
 	0,                         /* tp_iternext */
 	0,                         /* tp_methods */
 	0,                         /* tp_members */
 	0,                         /* tp_getset */
 	0,                         /* tp_base */
 	0,                         /* tp_dict */
 	0,                         /* tp_descr_get */
 	0,                         /* tp_descr_set */
 	0,                         /* tp_dictoffset */
 	0,                         /* tp_init */
 	0,                         /* tp_alloc */
 	dirstate_tuple_new,        /* tp_new */
 };
 
 static PyObject *parse_dirstate(PyObject *self, PyObject *args)
 {
 	PyObject *dmap, *cmap, *parents = NULL, *ret = NULL;
 	PyObject *fname = NULL, *cname = NULL, *entry = NULL;
 	char state, *cur, *str, *cpos;
 	int mode, size, mtime;
 	unsigned int flen, len, pos = 40;
 	int readlen;
 
 	if (!PyArg_ParseTuple(args, "O!O!s#:parse_dirstate",
 			      &PyDict_Type, &dmap,
 			      &PyDict_Type, &cmap,
 			      &str, &readlen))
 		goto quit;
 
 	len = readlen;
 
 	/* read parents */
 	if (len < 40) {
 		PyErr_SetString(
 			PyExc_ValueError, "too little data for parents");
 		goto quit;
 	}
 
 	parents = Py_BuildValue("s#s#", str, 20, str + 20, 20);
 	if (!parents)
 		goto quit;
 
 	/* read filenames */
 	while (pos >= 40 && pos < len) {
 		if (pos + 17 > len) {
 			PyErr_SetString(PyExc_ValueError,
 					"overflow in dirstate");
 			goto quit;
 		}
 		cur = str + pos;
 		/* unpack header */
 		state = *cur;
 		mode = getbe32(cur + 1);
 		size = getbe32(cur + 5);
 		mtime = getbe32(cur + 9);
 		flen = getbe32(cur + 13);
 		pos += 17;
 		cur += 17;
 		if (flen > len - pos) {
 			PyErr_SetString(PyExc_ValueError, "overflow in dirstate");
 			goto quit;
 		}
 
 		entry = (PyObject *)make_dirstate_tuple(state, mode, size,
 							mtime);
 		cpos = memchr(cur, 0, flen);
 		if (cpos) {
 			fname = PyBytes_FromStringAndSize(cur, cpos - cur);
 			cname = PyBytes_FromStringAndSize(cpos + 1,
 							   flen - (cpos - cur) - 1);
 			if (!fname || !cname ||
 			    PyDict_SetItem(cmap, fname, cname) == -1 ||
 			    PyDict_SetItem(dmap, fname, entry) == -1)
 				goto quit;
 			Py_DECREF(cname);
 		} else {
 			fname = PyBytes_FromStringAndSize(cur, flen);
 			if (!fname ||
 			    PyDict_SetItem(dmap, fname, entry) == -1)
 				goto quit;
 		}
 		Py_DECREF(fname);
 		Py_DECREF(entry);
 		fname = cname = entry = NULL;
 		pos += flen;
 	}
 
 	ret = parents;
 	Py_INCREF(ret);
 quit:
 	Py_XDECREF(fname);
 	Py_XDECREF(cname);
 	Py_XDECREF(entry);
 	Py_XDECREF(parents);
 	return ret;
 }
 
 /*
  * Build a set of non-normal and other parent entries from the dirstate dmap
 */
 static PyObject *nonnormalotherparententries(PyObject *self, PyObject *args) {
 	PyObject *dmap, *fname, *v;
 	PyObject *nonnset = NULL, *otherpset = NULL, *result = NULL;
 	Py_ssize_t pos;
 
 	if (!PyArg_ParseTuple(args, "O!:nonnormalentries",
 			      &PyDict_Type, &dmap))
 		goto bail;
 
 	nonnset = PySet_New(NULL);
 	if (nonnset == NULL)
 		goto bail;
 
 	otherpset = PySet_New(NULL);
 	if (otherpset == NULL)
 		goto bail;
 
 	pos = 0;
 	while (PyDict_Next(dmap, &pos, &fname, &v)) {
 		dirstateTupleObject *t;
 		if (!dirstate_tuple_check(v)) {
 			PyErr_SetString(PyExc_TypeError,
 					"expected a dirstate tuple");
 			goto bail;
 		}
 		t = (dirstateTupleObject *)v;
 
 		if (t->state == 'n' && t->size == -2) {
 			if (PySet_Add(otherpset, fname) == -1) {
 				goto bail;
 			}
 		}
 
 		if (t->state == 'n' && t->mtime != -1)
 			continue;
 		if (PySet_Add(nonnset, fname) == -1)
 			goto bail;
 	}
 
 	result = Py_BuildValue("(OO)", nonnset, otherpset);
 	if (result == NULL)
 		goto bail;
 	Py_DECREF(nonnset);
 	Py_DECREF(otherpset);
 	return result;
 bail:
 	Py_XDECREF(nonnset);
 	Py_XDECREF(otherpset);
 	Py_XDECREF(result);
 	return NULL;
 }
 
 /*
  * Efficiently pack a dirstate object into its on-disk format.
  */
 static PyObject *pack_dirstate(PyObject *self, PyObject *args)
 {
 	PyObject *packobj = NULL;
 	PyObject *map, *copymap, *pl, *mtime_unset = NULL;
 	Py_ssize_t nbytes, pos, l;
 	PyObject *k, *v = NULL, *pn;
 	char *p, *s;
 	int now;
 
 	if (!PyArg_ParseTuple(args, "O!O!Oi:pack_dirstate",
 			      &PyDict_Type, &map, &PyDict_Type, &copymap,
 			      &pl, &now))
 		return NULL;
 
 	if (!PySequence_Check(pl) || PySequence_Size(pl) != 2) {
 		PyErr_SetString(PyExc_TypeError, "expected 2-element sequence");
 		return NULL;
 	}
 
 	/* Figure out how much we need to allocate. */
 	for (nbytes = 40, pos = 0; PyDict_Next(map, &pos, &k, &v);) {
 		PyObject *c;
 		if (!PyBytes_Check(k)) {
 			PyErr_SetString(PyExc_TypeError, "expected string key");
 			goto bail;
 		}
 		nbytes += PyBytes_GET_SIZE(k) + 17;
 		c = PyDict_GetItem(copymap, k);
 		if (c) {
 			if (!PyBytes_Check(c)) {
 				PyErr_SetString(PyExc_TypeError,
 						"expected string key");
 				goto bail;
 			}
 			nbytes += PyBytes_GET_SIZE(c) + 1;
 		}
 	}
 
 	packobj = PyBytes_FromStringAndSize(NULL, nbytes);
 	if (packobj == NULL)
 		goto bail;
 
 	p = PyBytes_AS_STRING(packobj);
 
 	pn = PySequence_ITEM(pl, 0);
 	if (PyBytes_AsStringAndSize(pn, &s, &l) == -1 || l != 20) {
 		PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash");
 		goto bail;
 	}
 	memcpy(p, s, l);
 	p += 20;
 	pn = PySequence_ITEM(pl, 1);
 	if (PyBytes_AsStringAndSize(pn, &s, &l) == -1 || l != 20) {
 		PyErr_SetString(PyExc_TypeError, "expected a 20-byte hash");
 		goto bail;
 	}
 	memcpy(p, s, l);
 	p += 20;
 
 	for (pos = 0; PyDict_Next(map, &pos, &k, &v); ) {
 		dirstateTupleObject *tuple;
 		char state;
 		int mode, size, mtime;
 		Py_ssize_t len, l;
 		PyObject *o;
 		char *t;
 
 		if (!dirstate_tuple_check(v)) {
 			PyErr_SetString(PyExc_TypeError,
 					"expected a dirstate tuple");
 			goto bail;
 		}
 		tuple = (dirstateTupleObject *)v;
 
 		state = tuple->state;
 		mode = tuple->mode;
 		size = tuple->size;
 		mtime = tuple->mtime;
 		if (state == 'n' && mtime == now) {
 			/* See pure/parsers.py:pack_dirstate for why we do
 			 * this. */
 			mtime = -1;
 			mtime_unset = (PyObject *)make_dirstate_tuple(
 				state, mode, size, mtime);
 			if (!mtime_unset)
 				goto bail;
 			if (PyDict_SetItem(map, k, mtime_unset) == -1)
 				goto bail;
 			Py_DECREF(mtime_unset);
 			mtime_unset = NULL;
 		}
 		*p++ = state;
 		putbe32((uint32_t)mode, p);
 		putbe32((uint32_t)size, p + 4);
 		putbe32((uint32_t)mtime, p + 8);
 		t = p + 12;
 		p += 16;
 		len = PyBytes_GET_SIZE(k);
 		memcpy(p, PyBytes_AS_STRING(k), len);
 		p += len;
 		o = PyDict_GetItem(copymap, k);
 		if (o) {
 			*p++ = '\0';
 			l = PyBytes_GET_SIZE(o);
 			memcpy(p, PyBytes_AS_STRING(o), l);
 			p += l;
 			len += l + 1;
 		}
 		putbe32((uint32_t)len, t);
 	}
 
 	pos = p - PyBytes_AS_STRING(packobj);
 	if (pos != nbytes) {
 		PyErr_Format(PyExc_SystemError, "bad dirstate size: %ld != %ld",
                              (long)pos, (long)nbytes);
 		goto bail;
 	}
 
 	return packobj;
 bail:
 	Py_XDECREF(mtime_unset);
 	Py_XDECREF(packobj);
 	Py_XDECREF(v);
 	return NULL;
 }
 
 #define BUMPED_FIX 1
 #define USING_SHA_256 2
 #define FM1_HEADER_SIZE (4 + 8 + 2 + 2 + 1 + 1 + 1)
 
 static PyObject *readshas(
 	const char *source, unsigned char num, Py_ssize_t hashwidth)
 {
 	int i;
 	PyObject *list = PyTuple_New(num);
 	if (list == NULL) {
 		return NULL;
 	}
 	for (i = 0; i < num; i++) {
 		PyObject *hash = PyBytes_FromStringAndSize(source, hashwidth);
 		if (hash == NULL) {
 			Py_DECREF(list);
 			return NULL;
 		}
 		PyTuple_SET_ITEM(list, i, hash);
 		source += hashwidth;
 	}
 	return list;
 }
 
 static PyObject *fm1readmarker(const char *databegin, const char *dataend,
 			       uint32_t *msize)
 {
 	const char *data = databegin;
 	const char *meta;
 
 	double mtime;
 	int16_t tz;
 	uint16_t flags;
 	unsigned char nsuccs, nparents, nmetadata;
 	Py_ssize_t hashwidth = 20;
 
 	PyObject *prec = NULL, *parents = NULL, *succs = NULL;
 	PyObject *metadata = NULL, *ret = NULL;
 	int i;
 
 	if (data + FM1_HEADER_SIZE > dataend) {
 		goto overflow;
 	}
 
 	*msize = getbe32(data);
 	data += 4;
 	mtime = getbefloat64(data);
 	data += 8;
 	tz = getbeint16(data);
 	data += 2;
 	flags = getbeuint16(data);
 	data += 2;
 
 	if (flags & USING_SHA_256) {
 		hashwidth = 32;
 	}
 
 	nsuccs = (unsigned char)(*data++);
 	nparents = (unsigned char)(*data++);
 	nmetadata = (unsigned char)(*data++);
 
 	if (databegin + *msize > dataend) {
 		goto overflow;
 	}
 	dataend = databegin + *msize;  /* narrow down to marker size */
 
 	if (data + hashwidth > dataend) {
 		goto overflow;
 	}
 	prec = PyBytes_FromStringAndSize(data, hashwidth);
 	data += hashwidth;
 	if (prec == NULL) {
 		goto bail;
 	}
 
 	if (data + nsuccs * hashwidth > dataend) {
 		goto overflow;
 	}
 	succs = readshas(data, nsuccs, hashwidth);
 	if (succs == NULL) {
 		goto bail;
 	}
 	data += nsuccs * hashwidth;
 
 	if (nparents == 1 || nparents == 2) {
 		if (data + nparents * hashwidth > dataend) {
 			goto overflow;
 		}
 		parents = readshas(data, nparents, hashwidth);
 		if (parents == NULL) {
 			goto bail;
 		}
 		data += nparents * hashwidth;
 	} else {
 		parents = Py_None;
 		Py_INCREF(parents);
 	}
 
 	if (data + 2 * nmetadata > dataend) {
 		goto overflow;
 	}
 	meta = data + (2 * nmetadata);
 	metadata = PyTuple_New(nmetadata);
 	if (metadata == NULL) {
 		goto bail;
 	}
 	for (i = 0; i < nmetadata; i++) {
 		PyObject *tmp, *left = NULL, *right = NULL;
 		Py_ssize_t leftsize = (unsigned char)(*data++);
 		Py_ssize_t rightsize = (unsigned char)(*data++);
 		if (meta + leftsize + rightsize > dataend) {
 			goto overflow;
 		}
 		left = PyBytes_FromStringAndSize(meta, leftsize);
 		meta += leftsize;
 		right = PyBytes_FromStringAndSize(meta, rightsize);
 		meta += rightsize;
 		tmp = PyTuple_New(2);
 		if (!left || !right || !tmp) {
 			Py_XDECREF(left);
 			Py_XDECREF(right);
 			Py_XDECREF(tmp);
 			goto bail;
 		}
 		PyTuple_SET_ITEM(tmp, 0, left);
 		PyTuple_SET_ITEM(tmp, 1, right);
 		PyTuple_SET_ITEM(metadata, i, tmp);
 	}
 	ret = Py_BuildValue("(OOHO(di)O)", prec, succs, flags,
 			    metadata, mtime, (int)tz * 60, parents);
 	goto bail;  /* return successfully */
 
 overflow:
 	PyErr_SetString(PyExc_ValueError, "overflow in obsstore");
 bail:
 	Py_XDECREF(prec);
 	Py_XDECREF(succs);
 	Py_XDECREF(metadata);
 	Py_XDECREF(parents);
 	return ret;
 }
 
 
 static PyObject *fm1readmarkers(PyObject *self, PyObject *args) {
 	const char *data, *dataend;
 	int datalen;
 	Py_ssize_t offset, stop;
 	PyObject *markers = NULL;
 
 	if (!PyArg_ParseTuple(args, "s#nn", &data, &datalen, &offset, &stop)) {
 		return NULL;
 	}
 	dataend = data + datalen;
 	data += offset;
 	markers = PyList_New(0);
 	if (!markers) {
 		return NULL;
 	}
 	while (offset < stop) {
 		uint32_t msize;
 		int error;
 		PyObject *record = fm1readmarker(data, dataend, &msize);
 		if (!record) {
 			goto bail;
 		}
 		error = PyList_Append(markers, record);
 		Py_DECREF(record);
 		if (error) {
 			goto bail;
 		}
 		data += msize;
 		offset += msize;
 	}
 	return markers;
 bail:
 	Py_DECREF(markers);
 	return NULL;
 }
 
 static char parsers_doc[] = "Efficient content parsing.";
 
 PyObject *encodedir(PyObject *self, PyObject *args);
 PyObject *pathencode(PyObject *self, PyObject *args);
 PyObject *lowerencode(PyObject *self, PyObject *args);
 PyObject *parse_index2(PyObject *self, PyObject *args);
 
 static PyMethodDef methods[] = {
 	{"pack_dirstate", pack_dirstate, METH_VARARGS, "pack a dirstate\n"},
 	{"nonnormalotherparententries", nonnormalotherparententries, METH_VARARGS,
 	"create a set containing non-normal and other parent entries of given "
 	"dirstate\n"},
 	{"parse_manifest", parse_manifest, METH_VARARGS, "parse a manifest\n"},
 	{"parse_dirstate", parse_dirstate, METH_VARARGS, "parse a dirstate\n"},
 	{"parse_index2", parse_index2, METH_VARARGS, "parse a revlog index\n"},
 	{"asciilower", asciilower, METH_VARARGS, "lowercase an ASCII string\n"},
 	{"asciiupper", asciiupper, METH_VARARGS, "uppercase an ASCII string\n"},
 	{"dict_new_presized", dict_new_presized, METH_VARARGS,
 	 "construct a dict with an expected size\n"},
 	{"make_file_foldmap", make_file_foldmap, METH_VARARGS,
 	 "make file foldmap\n"},
 	{"encodedir", encodedir, METH_VARARGS, "encodedir a path\n"},
 	{"pathencode", pathencode, METH_VARARGS, "fncache-encode a path\n"},
 	{"lowerencode", lowerencode, METH_VARARGS, "lower-encode a path\n"},
 	{"fm1readmarkers", fm1readmarkers, METH_VARARGS,
 			"parse v1 obsolete markers\n"},
 	{NULL, NULL}
 };
 
 void dirs_module_init(PyObject *mod);
 void manifest_module_init(PyObject *mod);
 void revlog_module_init(PyObject *mod);
 
 static const int version = 1;
 
 static void module_init(PyObject *mod)
 {
 	PyModule_AddIntConstant(mod, "version", version);
 
 	/* This module constant has two purposes.  First, it lets us unit test
 	 * the ImportError raised without hard-coding any error text.  This
 	 * means we can change the text in the future without breaking tests,
 	 * even across changesets without a recompile.  Second, its presence
 	 * can be used to determine whether the version-checking logic is
 	 * present, which also helps in testing across changesets without a
 	 * recompile.  Note that this means the pure-Python version of parsers
 	 * should not have this module constant. */
 	PyModule_AddStringConstant(mod, "versionerrortext", versionerrortext);
 
 	dirs_module_init(mod);
 	manifest_module_init(mod);
 	revlog_module_init(mod);
 
 	if (PyType_Ready(&dirstateTupleType) < 0)
 		return;
 	Py_INCREF(&dirstateTupleType);
 	PyModule_AddObject(mod, "dirstatetuple",
 			   (PyObject *)&dirstateTupleType);
 }
 
 static int check_python_version(void)
 {
 	PyObject *sys = PyImport_ImportModule("sys"), *ver;
 	long hexversion;
 	if (!sys)
 		return -1;
 	ver = PyObject_GetAttrString(sys, "hexversion");
 	Py_DECREF(sys);
 	if (!ver)
 		return -1;
 	hexversion = PyInt_AsLong(ver);
 	Py_DECREF(ver);
 	/* sys.hexversion is a 32-bit number by default, so the -1 case
 	 * should only occur in unusual circumstances (e.g. if sys.hexversion
 	 * is manually set to an invalid value). */
 	if ((hexversion == -1) || (hexversion >> 16 != PY_VERSION_HEX >> 16)) {
 		PyErr_Format(PyExc_ImportError, "%s: The Mercurial extension "
 			"modules were compiled with Python " PY_VERSION ", but "
 			"Mercurial is currently using Python with sys.hexversion=%ld: "
 			"Python %s\n at: %s", versionerrortext, hexversion,
 			Py_GetVersion(), Py_GetProgramFullPath());
 		return -1;
 	}
 	return 0;
 }
 
 #ifdef IS_PY3K
 static struct PyModuleDef parsers_module = {
 	PyModuleDef_HEAD_INIT,
 	"parsers",
 	parsers_doc,
 	-1,
 	methods
 };
 
 PyMODINIT_FUNC PyInit_parsers(void)
 {
 	PyObject *mod;
 
 	if (check_python_version() == -1)
 		return NULL;
 	mod = PyModule_Create(&parsers_module);
 	module_init(mod);
 	return mod;
 }
 #else
 PyMODINIT_FUNC initparsers(void)
 {
 	PyObject *mod;
 
 	if (check_python_version() == -1)
 		return;
 	mod = Py_InitModule3("parsers", methods, parsers_doc);
 	module_init(mod);
 }
 #endif

setup.py

@@ -1,952 +1,953
 #
 # This is the mercurial setup script.
 #
 # 'python setup.py install', or
 # 'python setup.py --help' for more options
 
 import os
 
 supportedpy = '~= 2.7'
 if os.environ.get('HGALLOWPYTHON3', ''):
     # Mercurial will never work on Python 3 before 3.5 due to a lack
     # of % formatting on bytestrings, and can't work on 3.6.0 or 3.6.1
     # due to a bug in % formatting in bytestrings.
     #
     # TODO: when we actually work on Python 3, use this string as the
     # actual supportedpy string.
     supportedpy = ','.join([
         '>=2.7',
         '!=3.0.*',
         '!=3.1.*',
         '!=3.2.*',
         '!=3.3.*',
         '!=3.4.*',
         '!=3.6.0',
         '!=3.6.1',
     ])
 
 import sys, platform
 if sys.version_info[0] >= 3:
     printf = eval('print')
     libdir_escape = 'unicode_escape'
 else:
     libdir_escape = 'string_escape'
     def printf(*args, **kwargs):
         f = kwargs.get('file', sys.stdout)
         end = kwargs.get('end', '\n')
         f.write(b' '.join(args) + end)
 
 # Attempt to guide users to a modern pip - this means that 2.6 users
 # should have a chance of getting a 4.2 release, and when we ratchet
 # the version requirement forward again hopefully everyone will get
 # something that works for them.
 if sys.version_info < (2, 7, 0, 'final'):
     pip_message = ('This may be due to an out of date pip. '
                    'Make sure you have pip >= 9.0.1.')
     try:
         import pip
         pip_version = tuple([int(x) for x in pip.__version__.split('.')[:3]])
         if pip_version < (9, 0, 1) :
             pip_message = (
                 'Your pip version is out of date, please install '
                 'pip >= 9.0.1. pip {} detected.'.format(pip.__version__))
         else:
             # pip is new enough - it must be something else
             pip_message = ''
     except Exception:
         pass
     error = """
 Mercurial does not support Python older than 2.7.
 Python {py} detected.
 {pip}
 """.format(py=sys.version_info, pip=pip_message)
     printf(error, file=sys.stderr)
     sys.exit(1)
 
 # Solaris Python packaging brain damage
 try:
     import hashlib
     sha = hashlib.sha1()
 except ImportError:
     try:
         import sha
         sha.sha # silence unused import warning
     except ImportError:
         raise SystemExit(
             "Couldn't import standard hashlib (incomplete Python install).")
 
 try:
     import zlib
     zlib.compressobj # silence unused import warning
 except ImportError:
     raise SystemExit(
         "Couldn't import standard zlib (incomplete Python install).")
 
 # The base IronPython distribution (as of 2.7.1) doesn't support bz2
 isironpython = False
 try:
     isironpython = (platform.python_implementation()
                     .lower().find("ironpython") != -1)
 except AttributeError:
     pass
 
 if isironpython:
     sys.stderr.write("warning: IronPython detected (no bz2 support)\n")
 else:
     try:
         import bz2
         bz2.BZ2Compressor # silence unused import warning
     except ImportError:
         raise SystemExit(
             "Couldn't import standard bz2 (incomplete Python install).")
 
 ispypy = "PyPy" in sys.version
 
 import ctypes
 import stat, subprocess, time
 import re
 import shutil
 import tempfile
 from distutils import log
 # We have issues with setuptools on some platforms and builders. Until
 # those are resolved, setuptools is opt-in except for platforms where
 # we don't have issues.
 issetuptools = (os.name == 'nt' or 'FORCE_SETUPTOOLS' in os.environ)
 if issetuptools:
     from setuptools import setup
 else:
     from distutils.core import setup
 from distutils.ccompiler import new_compiler
 from distutils.core import Command, Extension
 from distutils.dist import Distribution
 from distutils.command.build import build
 from distutils.command.build_ext import build_ext
 from distutils.command.build_py import build_py
 from distutils.command.build_scripts import build_scripts
 from distutils.command.install import install
 from distutils.command.install_lib import install_lib
 from distutils.command.install_scripts import install_scripts
 from distutils.spawn import spawn, find_executable
 from distutils import file_util
 from distutils.errors import (
     CCompilerError,
     DistutilsError,
     DistutilsExecError,
 )
 from distutils.sysconfig import get_python_inc, get_config_var
 from distutils.version import StrictVersion
 
 scripts = ['hg']
 if os.name == 'nt':
     # We remove hg.bat if we are able to build hg.exe.
     scripts.append('contrib/win32/hg.bat')
 
 def cancompile(cc, code):
     tmpdir = tempfile.mkdtemp(prefix='hg-install-')
     devnull = oldstderr = None
     try:
         fname = os.path.join(tmpdir, 'testcomp.c')
         f = open(fname, 'w')
         f.write(code)
         f.close()
         # Redirect stderr to /dev/null to hide any error messages
         # from the compiler.
         # This will have to be changed if we ever have to check
         # for a function on Windows.
         devnull = open('/dev/null', 'w')
         oldstderr = os.dup(sys.stderr.fileno())
         os.dup2(devnull.fileno(), sys.stderr.fileno())
         objects = cc.compile([fname], output_dir=tmpdir)
         cc.link_executable(objects, os.path.join(tmpdir, "a.out"))
         return True
     except Exception:
         return False
     finally:
         if oldstderr is not None:
             os.dup2(oldstderr, sys.stderr.fileno())
         if devnull is not None:
             devnull.close()
         shutil.rmtree(tmpdir)
 
 # simplified version of distutils.ccompiler.CCompiler.has_function
 # that actually removes its temporary files.
 def hasfunction(cc, funcname):
     code = 'int main(void) { %s(); }\n' % funcname
     return cancompile(cc, code)
 
 def hasheader(cc, headername):
     code = '#include <%s>\nint main(void) { return 0; }\n' % headername
     return cancompile(cc, code)
 
 # py2exe needs to be installed to work
 try:
     import py2exe
     py2exe.Distribution # silence unused import warning
     py2exeloaded = True
     # import py2exe's patched Distribution class
     from distutils.core import Distribution
 except ImportError:
     py2exeloaded = False
 
 def runcmd(cmd, env):
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE, env=env)
     out, err = p.communicate()
     return p.returncode, out, err
 
 class hgcommand(object):
     def __init__(self, cmd, env):
         self.cmd = cmd
         self.env = env
 
     def run(self, args):
         cmd = self.cmd + args
         returncode, out, err = runcmd(cmd, self.env)
         err = filterhgerr(err)
         if err or returncode != 0:
             printf("stderr from '%s':" % (' '.join(cmd)), file=sys.stderr)
             printf(err, file=sys.stderr)
             return ''
         return out
 
 def filterhgerr(err):
     # If root is executing setup.py, but the repository is owned by
     # another user (as in "sudo python setup.py install") we will get
     # trust warnings since the .hg/hgrc file is untrusted. That is
     # fine, we don't want to load it anyway.  Python may warn about
     # a missing __init__.py in mercurial/locale, we also ignore that.
     err = [e for e in err.splitlines()
            if (not e.startswith(b'not trusting file')
                and not e.startswith(b'warning: Not importing')
                and not e.startswith(b'obsolete feature not enabled'))]
     return b'\n'.join(b'  ' + e for e in err)
 
 def findhg():
     """Try to figure out how we should invoke hg for examining the local
     repository contents.
 
     Returns an hgcommand object."""
     # By default, prefer the "hg" command in the user's path.  This was
     # presumably the hg command that the user used to create this repository.
     #
     # This repository may require extensions or other settings that would not
     # be enabled by running the hg script directly from this local repository.
     hgenv = os.environ.copy()
     # Use HGPLAIN to disable hgrc settings that would change output formatting,
     # and disable localization for the same reasons.
     hgenv['HGPLAIN'] = '1'
     hgenv['LANGUAGE'] = 'C'
     hgcmd = ['hg']
     # Run a simple "hg log" command just to see if using hg from the user's
     # path works and can successfully interact with this repository.
     check_cmd = ['log', '-r.', '-Ttest']
     try:
         retcode, out, err = runcmd(hgcmd + check_cmd, hgenv)
     except EnvironmentError:
         retcode = -1
     if retcode == 0 and not filterhgerr(err):
         return hgcommand(hgcmd, hgenv)
 
     # Fall back to trying the local hg installation.
     hgenv = localhgenv()
     # Don't source any system hgrc files when using the local hg.
     hgenv['HGRCPATH'] = ''
     hgcmd = [sys.executable, 'hg']
     try:
         retcode, out, err = runcmd(hgcmd + check_cmd, hgenv)
     except EnvironmentError:
         retcode = -1
     if retcode == 0 and not filterhgerr(err):
         return hgcommand(hgcmd, hgenv)
 
     raise SystemExit('Unable to find a working hg binary to extract the '
                      'version from the repository tags')
 
 def localhgenv():
     """Get an environment dictionary to use for invoking or importing
     mercurial from the local repository."""
     # Execute hg out of this directory with a custom environment which takes
     # care to not use any hgrc files and do no localization.
     env = {'HGMODULEPOLICY': 'py',
            'HGRCPATH': '',
            'LANGUAGE': 'C',
            'PATH': ''} # make pypi modules that use os.environ['PATH'] happy
     if 'LD_LIBRARY_PATH' in os.environ:
         env['LD_LIBRARY_PATH'] = os.environ['LD_LIBRARY_PATH']
     if 'SystemRoot' in os.environ:
         # SystemRoot is required by Windows to load various DLLs.  See:
         # https://bugs.python.org/issue13524#msg148850
         env['SystemRoot'] = os.environ['SystemRoot']
     return env
 
 version = ''
 
 if os.path.isdir('.hg'):
     hg = findhg()
     cmd = ['log', '-r', '.', '--template', '{tags}\n']
     numerictags = [t for t in hg.run(cmd).split() if t[0:1].isdigit()]
     hgid = hg.run(['id', '-i']).strip()
     if not hgid:
         # Bail out if hg is having problems interacting with this repository,
         # rather than falling through and producing a bogus version number.
         # Continuing with an invalid version number will break extensions
         # that define minimumhgversion.
         raise SystemExit('Unable to determine hg version from local repository')
     if numerictags: # tag(s) found
         version = numerictags[-1]
         if hgid.endswith('+'): # propagate the dirty status to the tag
             version += '+'
     else: # no tag found
         ltagcmd = ['parents', '--template', '{latesttag}']
         ltag = hg.run(ltagcmd)
         changessincecmd = ['log', '-T', 'x\n', '-r', "only(.,'%s')" % ltag]
         changessince = len(hg.run(changessincecmd).splitlines())
         version = '%s+%s-%s' % (ltag, changessince, hgid)
     if version.endswith('+'):
         version += time.strftime('%Y%m%d')
 elif os.path.exists('.hg_archival.txt'):
     kw = dict([[t.strip() for t in l.split(':', 1)]
                for l in open('.hg_archival.txt')])
     if 'tag' in kw:
         version = kw['tag']
     elif 'latesttag' in kw:
         if 'changessincelatesttag' in kw:
             version = '%(latesttag)s+%(changessincelatesttag)s-%(node).12s' % kw
         else:
             version = '%(latesttag)s+%(latesttagdistance)s-%(node).12s' % kw
     else:
         version = kw.get('node', '')[:12]
 
 if version:
     with open("mercurial/__version__.py", "w") as f:
         f.write('# this file is autogenerated by setup.py\n')
         f.write('version = "%s"\n' % version)
 
 try:
     oldpolicy = os.environ.get('HGMODULEPOLICY', None)
     os.environ['HGMODULEPOLICY'] = 'py'
     from mercurial import __version__
     version = __version__.version
 except ImportError:
     version = 'unknown'
 finally:
     if oldpolicy is None:
         del os.environ['HGMODULEPOLICY']
     else:
         os.environ['HGMODULEPOLICY'] = oldpolicy
 
 class hgbuild(build):
     # Insert hgbuildmo first so that files in mercurial/locale/ are found
     # when build_py is run next.
     sub_commands = [('build_mo', None)] + build.sub_commands
 
 class hgbuildmo(build):
 
     description = "build translations (.mo files)"
 
     def run(self):
         if not find_executable('msgfmt'):
             self.warn("could not find msgfmt executable, no translations "
                      "will be built")
             return
 
         podir = 'i18n'
         if not os.path.isdir(podir):
             self.warn("could not find %s/ directory" % podir)
             return
 
         join = os.path.join
         for po in os.listdir(podir):
             if not po.endswith('.po'):
                 continue
             pofile = join(podir, po)
             modir = join('locale', po[:-3], 'LC_MESSAGES')
             mofile = join(modir, 'hg.mo')
             mobuildfile = join('mercurial', mofile)
             cmd = ['msgfmt', '-v', '-o', mobuildfile, pofile]
             if sys.platform != 'sunos5':
                 # msgfmt on Solaris does not know about -c
                 cmd.append('-c')
             self.mkpath(join('mercurial', modir))
             self.make_file([pofile], mobuildfile, spawn, (cmd,))
 
 
 class hgdist(Distribution):
     pure = False
     cffi = ispypy
 
     global_options = Distribution.global_options + \
                      [('pure', None, "use pure (slow) Python "
                         "code instead of C extensions"),
                      ]
 
     def has_ext_modules(self):
         # self.ext_modules is emptied in hgbuildpy.finalize_options which is
         # too late for some cases
         return not self.pure and Distribution.has_ext_modules(self)
 
 # This is ugly as a one-liner. So use a variable.
 buildextnegops = dict(getattr(build_ext, 'negative_options', {}))
 buildextnegops['no-zstd'] = 'zstd'
 
 class hgbuildext(build_ext):
     user_options = build_ext.user_options + [
         ('zstd', None, 'compile zstd bindings [default]'),
         ('no-zstd', None, 'do not compile zstd bindings'),
     ]
 
     boolean_options = build_ext.boolean_options + ['zstd']
     negative_opt = buildextnegops
 
     def initialize_options(self):
         self.zstd = True
         return build_ext.initialize_options(self)
 
     def build_extensions(self):
         # Filter out zstd if disabled via argument.
         if not self.zstd:
             self.extensions = [e for e in self.extensions
                                if e.name != 'mercurial.zstd']
 
         return build_ext.build_extensions(self)
 
     def build_extension(self, ext):
         try:
             build_ext.build_extension(self, ext)
         except CCompilerError:
             if not getattr(ext, 'optional', False):
                 raise
             log.warn("Failed to build optional extension '%s' (skipping)",
                      ext.name)
 
 class hgbuildscripts(build_scripts):
     def run(self):
         if os.name != 'nt' or self.distribution.pure:
             return build_scripts.run(self)
 
         exebuilt = False
         try:
             self.run_command('build_hgexe')
             exebuilt = True
         except (DistutilsError, CCompilerError):
             log.warn('failed to build optional hg.exe')
 
         if exebuilt:
             # Copying hg.exe to the scripts build directory ensures it is
             # installed by the install_scripts command.
             hgexecommand = self.get_finalized_command('build_hgexe')
             dest = os.path.join(self.build_dir, 'hg.exe')
             self.mkpath(self.build_dir)
             self.copy_file(hgexecommand.hgexepath, dest)
 
             # Remove hg.bat because it is redundant with hg.exe.
             self.scripts.remove('contrib/win32/hg.bat')
 
         return build_scripts.run(self)
 
 class hgbuildpy(build_py):
     def finalize_options(self):
         build_py.finalize_options(self)
 
         if self.distribution.pure:
             self.distribution.ext_modules = []
         elif self.distribution.cffi:
             from mercurial.cffi import (
                 bdiffbuild,
                 mpatchbuild,
             )
             exts = [mpatchbuild.ffi.distutils_extension(),
                     bdiffbuild.ffi.distutils_extension()]
             # cffi modules go here
             if sys.platform == 'darwin':
                 from mercurial.cffi import osutilbuild
                 exts.append(osutilbuild.ffi.distutils_extension())
             self.distribution.ext_modules = exts
         else:
             h = os.path.join(get_python_inc(), 'Python.h')
             if not os.path.exists(h):
                 raise SystemExit('Python headers are required to build '
                                  'Mercurial but weren\'t found in %s' % h)
 
     def run(self):
         basepath = os.path.join(self.build_lib, 'mercurial')
         self.mkpath(basepath)
 
         if self.distribution.pure:
             modulepolicy = 'py'
         elif self.build_lib == '.':
             # in-place build should run without rebuilding C extensions
             modulepolicy = 'allow'
         else:
             modulepolicy = 'c'
         with open(os.path.join(basepath, '__modulepolicy__.py'), "w") as f:
             f.write('# this file is autogenerated by setup.py\n')
             f.write('modulepolicy = b"%s"\n' % modulepolicy)
 
         build_py.run(self)
 
 class buildhgextindex(Command):
     description = 'generate prebuilt index of hgext (for frozen package)'
     user_options = []
     _indexfilename = 'hgext/__index__.py'
 
     def initialize_options(self):
         pass
 
     def finalize_options(self):
         pass
 
     def run(self):
         if os.path.exists(self._indexfilename):
             with open(self._indexfilename, 'w') as f:
                 f.write('# empty\n')
 
         # here no extension enabled, disabled() lists up everything
         code = ('import pprint; from mercurial import extensions; '
                 'pprint.pprint(extensions.disabled())')
         returncode, out, err = runcmd([sys.executable, '-c', code],
                                       localhgenv())
         if err or returncode != 0:
             raise DistutilsExecError(err)
 
         with open(self._indexfilename, 'w') as f:
             f.write('# this file is autogenerated by setup.py\n')
             f.write('docs = ')
             f.write(out)
 
 class buildhgexe(build_ext):
     description = 'compile hg.exe from mercurial/exewrapper.c'
 
     def build_extensions(self):
         if os.name != 'nt':
             return
         if isinstance(self.compiler, HackedMingw32CCompiler):
             self.compiler.compiler_so = self.compiler.compiler # no -mdll
             self.compiler.dll_libraries = [] # no -lmsrvc90
 
         # Different Python installs can have different Python library
         # names. e.g. the official CPython distribution uses pythonXY.dll
         # and MinGW uses libpythonX.Y.dll.
         _kernel32 = ctypes.windll.kernel32
         _kernel32.GetModuleFileNameA.argtypes = [ctypes.c_void_p,
                                                  ctypes.c_void_p,
                                                  ctypes.c_ulong]
         _kernel32.GetModuleFileNameA.restype = ctypes.c_ulong
         size = 1000
         buf = ctypes.create_string_buffer(size + 1)
         filelen = _kernel32.GetModuleFileNameA(sys.dllhandle, ctypes.byref(buf),
                                                size)
 
         if filelen > 0 and filelen != size:
             dllbasename = os.path.basename(buf.value)
             if not dllbasename.lower().endswith('.dll'):
                 raise SystemExit('Python DLL does not end with .dll: %s' %
                                  dllbasename)
             pythonlib = dllbasename[:-4]
         else:
             log.warn('could not determine Python DLL filename; '
                      'assuming pythonXY')
 
             hv = sys.hexversion
             pythonlib = 'python%d%d' % (hv >> 24, (hv >> 16) & 0xff)
 
         log.info('using %s as Python library name' % pythonlib)
         with open('mercurial/hgpythonlib.h', 'wb') as f:
             f.write('/* this file is autogenerated by setup.py */\n')
             f.write('#define HGPYTHONLIB "%s"\n' % pythonlib)
         objects = self.compiler.compile(['mercurial/exewrapper.c'],
                                          output_dir=self.build_temp)
         dir = os.path.dirname(self.get_ext_fullpath('dummy'))
         target = os.path.join(dir, 'hg')
         self.compiler.link_executable(objects, target,
                                       libraries=[],
                                       output_dir=self.build_temp)
 
     @property
     def hgexepath(self):
         dir = os.path.dirname(self.get_ext_fullpath('dummy'))
         return os.path.join(self.build_temp, dir, 'hg.exe')
 
 class hginstall(install):
 
     user_options = install.user_options + [
         ('old-and-unmanageable', None,
          'noop, present for eggless setuptools compat'),
         ('single-version-externally-managed', None,
          'noop, present for eggless setuptools compat'),
     ]
 
     # Also helps setuptools not be sad while we refuse to create eggs.
     single_version_externally_managed = True
 
     def get_sub_commands(self):
         # Screen out egg related commands to prevent egg generation.  But allow
         # mercurial.egg-info generation, since that is part of modern
         # packaging.
         excl = set(['bdist_egg'])
         return filter(lambda x: x not in excl, install.get_sub_commands(self))
 
 class hginstalllib(install_lib):
     '''
     This is a specialization of install_lib that replaces the copy_file used
     there so that it supports setting the mode of files after copying them,
     instead of just preserving the mode that the files originally had.  If your
     system has a umask of something like 027, preserving the permissions when
     copying will lead to a broken install.
 
     Note that just passing keep_permissions=False to copy_file would be
     insufficient, as it might still be applying a umask.
     '''
 
     def run(self):
         realcopyfile = file_util.copy_file
         def copyfileandsetmode(*args, **kwargs):
             src, dst = args[0], args[1]
             dst, copied = realcopyfile(*args, **kwargs)
             if copied:
                 st = os.stat(src)
                 # Persist executable bit (apply it to group and other if user
                 # has it)
                 if st[stat.ST_MODE] & stat.S_IXUSR:
                     setmode = int('0755', 8)
                 else:
                     setmode = int('0644', 8)
                 m = stat.S_IMODE(st[stat.ST_MODE])
                 m = (m & ~int('0777', 8)) | setmode
                 os.chmod(dst, m)
         file_util.copy_file = copyfileandsetmode
         try:
             install_lib.run(self)
         finally:
             file_util.copy_file = realcopyfile
 
 class hginstallscripts(install_scripts):
     '''
     This is a specialization of install_scripts that replaces the @LIBDIR@ with
     the configured directory for modules. If possible, the path is made relative
     to the directory for scripts.
     '''
 
     def initialize_options(self):
         install_scripts.initialize_options(self)
 
         self.install_lib = None
 
     def finalize_options(self):
         install_scripts.finalize_options(self)
         self.set_undefined_options('install',
                                    ('install_lib', 'install_lib'))
 
     def run(self):
         install_scripts.run(self)
 
         # It only makes sense to replace @LIBDIR@ with the install path if
         # the install path is known. For wheels, the logic below calculates
         # the libdir to be "../..". This is because the internal layout of a
         # wheel archive looks like:
         #
         #   mercurial-3.6.1.data/scripts/hg
         #   mercurial/__init__.py
         #
         # When installing wheels, the subdirectories of the "<pkg>.data"
         # directory are translated to system local paths and files therein
         # are copied in place. The mercurial/* files are installed into the
         # site-packages directory. However, the site-packages directory
         # isn't known until wheel install time. This means we have no clue
         # at wheel generation time what the installed site-packages directory
         # will be. And, wheels don't appear to provide the ability to register
         # custom code to run during wheel installation. This all means that
         # we can't reliably set the libdir in wheels: the default behavior
         # of looking in sys.path must do.
 
         if (os.path.splitdrive(self.install_dir)[0] !=
             os.path.splitdrive(self.install_lib)[0]):
             # can't make relative paths from one drive to another, so use an
             # absolute path instead
             libdir = self.install_lib
         else:
             common = os.path.commonprefix((self.install_dir, self.install_lib))
             rest = self.install_dir[len(common):]
             uplevel = len([n for n in os.path.split(rest) if n])
 
             libdir = uplevel * ('..' + os.sep) + self.install_lib[len(common):]
 
         for outfile in self.outfiles:
             with open(outfile, 'rb') as fp:
                 data = fp.read()
 
             # skip binary files
             if b'\0' in data:
                 continue
 
             # During local installs, the shebang will be rewritten to the final
             # install path. During wheel packaging, the shebang has a special
             # value.
             if data.startswith(b'#!python'):
                 log.info('not rewriting @LIBDIR@ in %s because install path '
                          'not known' % outfile)
                 continue
 
             data = data.replace(b'@LIBDIR@', libdir.encode(libdir_escape))
             with open(outfile, 'wb') as fp:
                 fp.write(data)
 
 cmdclass = {'build': hgbuild,
             'build_mo': hgbuildmo,
             'build_ext': hgbuildext,
             'build_py': hgbuildpy,
             'build_scripts': hgbuildscripts,
             'build_hgextindex': buildhgextindex,
             'install': hginstall,
             'install_lib': hginstalllib,
             'install_scripts': hginstallscripts,
             'build_hgexe': buildhgexe,
             }
 
 packages = ['mercurial',
             'mercurial.cext',
             'mercurial.cffi',
             'mercurial.hgweb',
             'mercurial.httpclient',
             'mercurial.pure',
             'hgext', 'hgext.convert', 'hgext.fsmonitor',
             'hgext.fsmonitor.pywatchman', 'hgext.highlight',
             'hgext.largefiles', 'hgext.zeroconf', 'hgext3rd',
             'hgdemandimport']
 
 common_depends = ['mercurial/bitmanipulation.h',
                   'mercurial/compat.h',
                   'mercurial/cext/util.h']
 common_include_dirs = ['mercurial']
 
 osutil_cflags = []
 osutil_ldflags = []
 
 # platform specific macros
 for plat, func in [('bsd', 'setproctitle')]:
     if re.search(plat, sys.platform) and hasfunction(new_compiler(), func):
         osutil_cflags.append('-DHAVE_%s' % func.upper())
 
 for plat, macro, code in [
     ('bsd|darwin', 'BSD_STATFS', '''
      #include <sys/param.h>
      #include <sys/mount.h>
      int main() { struct statfs s; return sizeof(s.f_fstypename); }
      '''),
     ('linux', 'LINUX_STATFS', '''
      #include <linux/magic.h>
      #include <sys/vfs.h>
      int main() { struct statfs s; return sizeof(s.f_type); }
      '''),
 ]:
     if re.search(plat, sys.platform) and cancompile(new_compiler(), code):
         osutil_cflags.append('-DHAVE_%s' % macro)
 
 if sys.platform == 'darwin':
     osutil_ldflags += ['-framework', 'ApplicationServices']
 
 extmodules = [
     Extension('mercurial.cext.base85', ['mercurial/cext/base85.c'],
               include_dirs=common_include_dirs,
               depends=common_depends),
     Extension('mercurial.cext.bdiff', ['mercurial/bdiff.c',
                                        'mercurial/cext/bdiff.c'],
               include_dirs=common_include_dirs,
               depends=common_depends + ['mercurial/bdiff.h']),
     Extension('mercurial.cext.diffhelpers', ['mercurial/cext/diffhelpers.c'],
               include_dirs=common_include_dirs,
               depends=common_depends),
     Extension('mercurial.cext.mpatch', ['mercurial/mpatch.c',
                                         'mercurial/cext/mpatch.c'],
               include_dirs=common_include_dirs,
               depends=common_depends),
-    Extension('mercurial.cext.parsers', ['mercurial/cext/dirs.c',
+    Extension('mercurial.cext.parsers', ['mercurial/cext/charencode.c',
+                                         'mercurial/cext/dirs.c',
                                          'mercurial/cext/manifest.c',
                                          'mercurial/cext/parsers.c',
                                          'mercurial/cext/pathencode.c',
                                          'mercurial/cext/revlog.c'],
               include_dirs=common_include_dirs,
               depends=common_depends),
     Extension('mercurial.cext.osutil', ['mercurial/cext/osutil.c'],
               include_dirs=common_include_dirs,
               extra_compile_args=osutil_cflags,
               extra_link_args=osutil_ldflags,
               depends=common_depends),
     Extension('hgext.fsmonitor.pywatchman.bser',
               ['hgext/fsmonitor/pywatchman/bser.c']),
     ]
 
 sys.path.insert(0, 'contrib/python-zstandard')
 import setup_zstd
 extmodules.append(setup_zstd.get_c_extension(name='mercurial.zstd'))
 
 try:
     from distutils import cygwinccompiler
 
     # the -mno-cygwin option has been deprecated for years
     compiler = cygwinccompiler.Mingw32CCompiler
 
     class HackedMingw32CCompiler(cygwinccompiler.Mingw32CCompiler):
         def __init__(self, *args, **kwargs):
             compiler.__init__(self, *args, **kwargs)
             for i in 'compiler compiler_so linker_exe linker_so'.split():
                 try:
                     getattr(self, i).remove('-mno-cygwin')
                 except ValueError:
                     pass
 
     cygwinccompiler.Mingw32CCompiler = HackedMingw32CCompiler
 except ImportError:
     # the cygwinccompiler package is not available on some Python
     # distributions like the ones from the optware project for Synology
     # DiskStation boxes
     class HackedMingw32CCompiler(object):
         pass
 
 if os.name == 'nt':
     # Allow compiler/linker flags to be added to Visual Studio builds.  Passing
     # extra_link_args to distutils.extensions.Extension() doesn't have any
     # effect.
     from distutils import msvccompiler
 
     compiler = msvccompiler.MSVCCompiler
 
     class HackedMSVCCompiler(msvccompiler.MSVCCompiler):
         def initialize(self):
             compiler.initialize(self)
             # "warning LNK4197: export 'func' specified multiple times"
             self.ldflags_shared.append('/ignore:4197')
             self.ldflags_shared_debug.append('/ignore:4197')
 
     msvccompiler.MSVCCompiler = HackedMSVCCompiler
 
 packagedata = {'mercurial': ['locale/*/LC_MESSAGES/hg.mo',
                              'help/*.txt',
                              'help/internals/*.txt',
                              'default.d/*.rc',
                              'dummycert.pem']}
 
 def ordinarypath(p):
     return p and p[0] != '.' and p[-1] != '~'
 
 for root in ('templates',):
     for curdir, dirs, files in os.walk(os.path.join('mercurial', root)):
         curdir = curdir.split(os.sep, 1)[1]
         dirs[:] = filter(ordinarypath, dirs)
         for f in filter(ordinarypath, files):
             f = os.path.join(curdir, f)
             packagedata['mercurial'].append(f)
 
 datafiles = []
 
 # distutils expects version to be str/unicode. Converting it to
 # unicode on Python 2 still works because it won't contain any
 # non-ascii bytes and will be implicitly converted back to bytes
 # when operated on.
 assert isinstance(version, bytes)
 setupversion = version.decode('ascii')
 
 extra = {}
 
 if issetuptools:
     extra['python_requires'] = supportedpy
 if py2exeloaded:
     extra['console'] = [
         {'script':'hg',
          'copyright':'Copyright (C) 2005-2017 Matt Mackall and others',
          'product_version':version}]
     # sub command of 'build' because 'py2exe' does not handle sub_commands
     build.sub_commands.insert(0, ('build_hgextindex', None))
     # put dlls in sub directory so that they won't pollute PATH
     extra['zipfile'] = 'lib/library.zip'
 
 if os.name == 'nt':
     # Windows binary file versions for exe/dll files must have the
     # form W.X.Y.Z, where W,X,Y,Z are numbers in the range 0..65535
     setupversion = version.split('+', 1)[0]
 
 if sys.platform == 'darwin' and os.path.exists('/usr/bin/xcodebuild'):
     version = runcmd(['/usr/bin/xcodebuild', '-version'], {})[1].splitlines()
     if version:
         version = version[0]
         if sys.version_info[0] == 3:
             version = version.decode('utf-8')
         xcode4 = (version.startswith('Xcode') and
                   StrictVersion(version.split()[1]) >= StrictVersion('4.0'))
         xcode51 = re.match(r'^Xcode\s+5\.1', version) is not None
     else:
         # xcodebuild returns empty on OS X Lion with XCode 4.3 not
         # installed, but instead with only command-line tools. Assume
         # that only happens on >= Lion, thus no PPC support.
         xcode4 = True
         xcode51 = False
 
     # XCode 4.0 dropped support for ppc architecture, which is hardcoded in
     # distutils.sysconfig
     if xcode4:
         os.environ['ARCHFLAGS'] = ''
 
     # XCode 5.1 changes clang such that it now fails to compile if the
     # -mno-fused-madd flag is passed, but the version of Python shipped with
     # OS X 10.9 Mavericks includes this flag. This causes problems in all
     # C extension modules, and a bug has been filed upstream at
     # http://bugs.python.org/issue21244. We also need to patch this here
     # so Mercurial can continue to compile in the meantime.
     if xcode51:
         cflags = get_config_var('CFLAGS')
         if cflags and re.search(r'-mno-fused-madd\b', cflags) is not None:
             os.environ['CFLAGS'] = (
                 os.environ.get('CFLAGS', '') + ' -Qunused-arguments')
 
 setup(name='mercurial',
       version=setupversion,
       author='Matt Mackall and many others',
       author_email='[e-mail]',
       url='https://mercurial-scm.org/',
       download_url='https://mercurial-scm.org/release/',
       description=('Fast scalable distributed SCM (revision control, version '
                    'control) system'),
       long_description=('Mercurial is a distributed SCM tool written in Python.'
                         ' It is used by a number of large projects that require'
                         ' fast, reliable distributed revision control, such as '
                         'Mozilla.'),
       license='GNU GPLv2 or any later version',
       classifiers=[
           'Development Status :: 6 - Mature',
           'Environment :: Console',
           'Intended Audience :: Developers',
           'Intended Audience :: System Administrators',
           'License :: OSI Approved :: GNU General Public License (GPL)',
           'Natural Language :: Danish',
           'Natural Language :: English',
           'Natural Language :: German',
           'Natural Language :: Italian',
           'Natural Language :: Japanese',
           'Natural Language :: Portuguese (Brazilian)',
           'Operating System :: Microsoft :: Windows',
           'Operating System :: OS Independent',
           'Operating System :: POSIX',
           'Programming Language :: C',
           'Programming Language :: Python',
           'Topic :: Software Development :: Version Control',
       ],
       scripts=scripts,
       packages=packages,
       ext_modules=extmodules,
       data_files=datafiles,
       package_data=packagedata,
       cmdclass=cmdclass,
       distclass=hgdist,
       options={'py2exe': {'packages': ['hgdemandimport', 'hgext', 'email',
                                        # implicitly imported per module policy
                                        # (cffi wouldn't be used as a frozen exe)
                                        'mercurial.cext',
                                        #'mercurial.cffi',
                                        'mercurial.pure']},
                'bdist_mpkg': {'zipdist': False,
                               'license': 'COPYING',
                               'readme': 'contrib/macosx/Readme.html',
                               'welcome': 'contrib/macosx/Welcome.html',
                               },
                },
       **extra)