manifest: make sure there's a filename before bothering to look for newline

There's no valid manifest that would have no characters before the NUL byte on
a line, and this fixes some erratic timeouts in the fuzzer.

Differential Revision: https://phab.mercurial-scm.org/D5256

mercurial/cext/manifest.c

@@ -1,960 +1,972
 /*
  * manifest.c - manifest type that does on-demand parsing.
  *
  * Copyright 2015, Google Inc.
  *
  * This software may be used and distributed according to the terms of
  * the GNU General Public License, incorporated herein by reference.
  */
 #include <Python.h>
 
 #include <assert.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "charencode.h"
 #include "util.h"
 
 #define DEFAULT_LINES 100000
 
 typedef struct {
 	char *start;
 	Py_ssize_t len; /* length of line including terminal newline */
 	char hash_suffix;
 	bool from_malloc;
 	bool deleted;
 } line;
 
 typedef struct {
 	PyObject_HEAD
 	PyObject *pydata;
 	line *lines;
 	int numlines; /* number of line entries */
 	int livelines; /* number of non-deleted lines */
 	int maxlines; /* allocated number of lines */
 	bool dirty;
 } lazymanifest;
 
 #define MANIFEST_OOM -1
 #define MANIFEST_NOT_SORTED -2
 #define MANIFEST_MALFORMED -3
+#define MANIFEST_BOGUS_FILENAME -4
 
 /* get the length of the path for a line */
 static size_t pathlen(line *l)
 {
 	const char *end = memchr(l->start, '\0', l->len);
 	return (end) ? (size_t)(end - l->start) : l->len;
 }
 
 /* get the node value of a single line */
 static PyObject *nodeof(line *l)
 {
 	char *s = l->start;
 	ssize_t llen = pathlen(l);
 	PyObject *hash;
 	if (llen + 1 + 40 + 1 > l->len) { /* path '\0' hash '\n' */
 		PyErr_SetString(PyExc_ValueError, "manifest line too short");
 		return NULL;
 	}
 	hash = unhexlify(s + llen + 1, 40);
 	if (!hash) {
 		return NULL;
 	}
 	if (l->hash_suffix != '\0') {
 		char newhash[21];
 		memcpy(newhash, PyBytes_AsString(hash), 20);
 		Py_DECREF(hash);
 		newhash[20] = l->hash_suffix;
 		hash = PyBytes_FromStringAndSize(newhash, 21);
 	}
 	return hash;
 }
 
 /* get the node hash and flags of a line as a tuple */
 static PyObject *hashflags(line *l)
 {
 	char *s = l->start;
 	size_t plen = pathlen(l);
 	PyObject *hash = nodeof(l);
 
 	/* 40 for hash, 1 for null byte, 1 for newline */
 	size_t hplen = plen + 42;
 	Py_ssize_t flen = l->len - hplen;
 	PyObject *flags;
 	PyObject *tup;
 
 	if (!hash)
 		return NULL;
 	flags = PyBytes_FromStringAndSize(s + hplen - 1, flen);
 	if (!flags) {
 		Py_DECREF(hash);
 		return NULL;
 	}
 	tup = PyTuple_Pack(2, hash, flags);
 	Py_DECREF(flags);
 	Py_DECREF(hash);
 	return tup;
 }
 
 /* if we're about to run out of space in the line index, add more */
 static bool realloc_if_full(lazymanifest *self)
 {
 	if (self->numlines == self->maxlines) {
 		self->maxlines *= 2;
 		self->lines = realloc(self->lines, self->maxlines * sizeof(line));
 	}
 	return !!self->lines;
 }
 
 /*
  * Find the line boundaries in the manifest that 'data' points to and store
  * information about each line in 'self'.
  */
 static int find_lines(lazymanifest *self, char *data, Py_ssize_t len)
 {
 	char *prev = NULL;
 	while (len > 0) {
 		line *l;
-		char *next = memchr(data, '\n', len);
+		char *next;
+		if (*data == '\0') {
+			/* It's implausible there's no filename, don't
+			 * even bother looking for the newline. */
+			return MANIFEST_BOGUS_FILENAME;
+		}
+		next = memchr(data, '\n', len);
 		if (!next) {
 			return MANIFEST_MALFORMED;
 		}
 		next++; /* advance past newline */
 		if (!realloc_if_full(self)) {
 			return MANIFEST_OOM; /* no memory */
 		}
 		if (prev && strcmp(prev, data) > -1) {
 			/* This data isn't sorted, so we have to abort. */
 			return MANIFEST_NOT_SORTED;
 		}
 		l = self->lines + ((self->numlines)++);
 		l->start = data;
 		l->len = next - data;
 		l->hash_suffix = '\0';
 		l->from_malloc = false;
 		l->deleted = false;
 		len = len - l->len;
 		prev = data;
 		data = next;
 	}
 	self->livelines = self->numlines;
 	return 0;
 }
 
 static void lazymanifest_init_early(lazymanifest *self)
 {
 	self->pydata = NULL;
 	self->lines = NULL;
 	self->numlines = 0;
 	self->maxlines = 0;
 }
 
 static int lazymanifest_init(lazymanifest *self, PyObject *args)
 {
 	char *data;
 	Py_ssize_t len;
 	int err, ret;
 	PyObject *pydata;
 
 	lazymanifest_init_early(self);
 	if (!PyArg_ParseTuple(args, "S", &pydata)) {
 		return -1;
 	}
 	err = PyBytes_AsStringAndSize(pydata, &data, &len);
 
 	self->dirty = false;
 	if (err == -1)
 		return -1;
 	self->pydata = pydata;
 	Py_INCREF(self->pydata);
 	Py_BEGIN_ALLOW_THREADS
 	self->lines = malloc(DEFAULT_LINES * sizeof(line));
 	self->maxlines = DEFAULT_LINES;
 	self->numlines = 0;
 	if (!self->lines)
 		ret = MANIFEST_OOM;
 	else
 		ret = find_lines(self, data, len);
 	Py_END_ALLOW_THREADS
 	switch (ret) {
 	case 0:
 		break;
 	case MANIFEST_OOM:
 		PyErr_NoMemory();
 		break;
 	case MANIFEST_NOT_SORTED:
 		PyErr_Format(PyExc_ValueError,
 			     "Manifest lines not in sorted order.");
 		break;
 	case MANIFEST_MALFORMED:
 		PyErr_Format(PyExc_ValueError,
 			     "Manifest did not end in a newline.");
 		break;
+	case MANIFEST_BOGUS_FILENAME:
+		PyErr_Format(
+			PyExc_ValueError,
+			"Manifest had an entry with a zero-length filename.");
+		break;
 	default:
 		PyErr_Format(PyExc_ValueError,
 			     "Unknown problem parsing manifest.");
 	}
 	return ret == 0 ? 0 : -1;
 }
 
 static void lazymanifest_dealloc(lazymanifest *self)
 {
 	/* free any extra lines we had to allocate */
 	int i;
 	for (i = 0; self->lines && (i < self->numlines); i++) {
 		if (self->lines[i].from_malloc) {
 			free(self->lines[i].start);
 		}
 	}
 	free(self->lines);
 	self->lines = NULL;
 	if (self->pydata) {
 		Py_DECREF(self->pydata);
 		self->pydata = NULL;
 	}
 	PyObject_Del(self);
 }
 
 /* iteration support */
 
 typedef struct {
 	PyObject_HEAD lazymanifest *m;
 	Py_ssize_t pos;
 } lmIter;
 
 static void lmiter_dealloc(PyObject *o)
 {
 	lmIter *self = (lmIter *)o;
 	Py_DECREF(self->m);
 	PyObject_Del(self);
 }
 
 static line *lmiter_nextline(lmIter *self)
 {
 	do {
 		self->pos++;
 		if (self->pos >= self->m->numlines) {
 			return NULL;
 		}
 		/* skip over deleted manifest entries */
 	} while (self->m->lines[self->pos].deleted);
 	return self->m->lines + self->pos;
 }
 
 static PyObject *lmiter_iterentriesnext(PyObject *o)
 {
 	size_t pl;
 	line *l;
 	Py_ssize_t consumed;
 	PyObject *ret = NULL, *path = NULL, *hash = NULL, *flags = NULL;
 	l = lmiter_nextline((lmIter *)o);
 	if (!l) {
 		goto done;
 	}
 	pl = pathlen(l);
 	path = PyBytes_FromStringAndSize(l->start, pl);
 	hash = nodeof(l);
 	if (!path || !hash) {
 		goto done;
 	}
 	consumed = pl + 41;
 	flags = PyBytes_FromStringAndSize(l->start + consumed,
 					   l->len - consumed - 1);
 	if (!flags) {
 		goto done;
 	}
 	ret = PyTuple_Pack(3, path, hash, flags);
 done:
 	Py_XDECREF(path);
 	Py_XDECREF(hash);
 	Py_XDECREF(flags);
 	return ret;
 }
 
 #ifdef IS_PY3K
 #define LAZYMANIFESTENTRIESITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT
 #else
 #define LAZYMANIFESTENTRIESITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT \
 	| Py_TPFLAGS_HAVE_ITER
 #endif
 
 static PyTypeObject lazymanifestEntriesIterator = {
 	PyVarObject_HEAD_INIT(NULL, 0) /* header */
 	"parsers.lazymanifest.entriesiterator", /*tp_name */
 	sizeof(lmIter),                  /*tp_basicsize */
 	0,                               /*tp_itemsize */
 	lmiter_dealloc,                  /*tp_dealloc */
 	0,                               /*tp_print */
 	0,                               /*tp_getattr */
 	0,                               /*tp_setattr */
 	0,                               /*tp_compare */
 	0,                               /*tp_repr */
 	0,                               /*tp_as_number */
 	0,                               /*tp_as_sequence */
 	0,                               /*tp_as_mapping */
 	0,                               /*tp_hash */
 	0,                               /*tp_call */
 	0,                               /*tp_str */
 	0,                               /*tp_getattro */
 	0,                               /*tp_setattro */
 	0,                               /*tp_as_buffer */
 	LAZYMANIFESTENTRIESITERATOR_TPFLAGS, /* tp_flags */
 	"Iterator for 3-tuples in a lazymanifest.",  /* tp_doc */
 	0,                               /* tp_traverse */
 	0,                               /* tp_clear */
 	0,                               /* tp_richcompare */
 	0,                               /* tp_weaklistoffset */
 	PyObject_SelfIter,               /* tp_iter: __iter__() method */
 	lmiter_iterentriesnext,          /* tp_iternext: next() method */
 };
 
 static PyObject *lmiter_iterkeysnext(PyObject *o)
 {
 	size_t pl;
 	line *l = lmiter_nextline((lmIter *)o);
 	if (!l) {
 		return NULL;
 	}
 	pl = pathlen(l);
 	return PyBytes_FromStringAndSize(l->start, pl);
 }
 
 #ifdef IS_PY3K
 #define LAZYMANIFESTKEYSITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT
 #else
 #define LAZYMANIFESTKEYSITERATOR_TPFLAGS Py_TPFLAGS_DEFAULT \
 	| Py_TPFLAGS_HAVE_ITER
 #endif
 
 static PyTypeObject lazymanifestKeysIterator = {
 	PyVarObject_HEAD_INIT(NULL, 0) /* header */
 	"parsers.lazymanifest.keysiterator", /*tp_name */
 	sizeof(lmIter),                  /*tp_basicsize */
 	0,                               /*tp_itemsize */
 	lmiter_dealloc,                  /*tp_dealloc */
 	0,                               /*tp_print */
 	0,                               /*tp_getattr */
 	0,                               /*tp_setattr */
 	0,                               /*tp_compare */
 	0,                               /*tp_repr */
 	0,                               /*tp_as_number */
 	0,                               /*tp_as_sequence */
 	0,                               /*tp_as_mapping */
 	0,                               /*tp_hash */
 	0,                               /*tp_call */
 	0,                               /*tp_str */
 	0,                               /*tp_getattro */
 	0,                               /*tp_setattro */
 	0,                               /*tp_as_buffer */
 	LAZYMANIFESTKEYSITERATOR_TPFLAGS, /* tp_flags */
 	"Keys iterator for a lazymanifest.",  /* tp_doc */
 	0,                               /* tp_traverse */
 	0,                               /* tp_clear */
 	0,                               /* tp_richcompare */
 	0,                               /* tp_weaklistoffset */
 	PyObject_SelfIter,               /* tp_iter: __iter__() method */
 	lmiter_iterkeysnext,             /* tp_iternext: next() method */
 };
 
 static lazymanifest *lazymanifest_copy(lazymanifest *self);
 
 static PyObject *lazymanifest_getentriesiter(lazymanifest *self)
 {
 	lmIter *i = NULL;
 	lazymanifest *t = lazymanifest_copy(self);
 	if (!t) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 	i = PyObject_New(lmIter, &lazymanifestEntriesIterator);
 	if (i) {
 		i->m = t;
 		i->pos = -1;
 	} else {
 		Py_DECREF(t);
 		PyErr_NoMemory();
 	}
 	return (PyObject *)i;
 }
 
 static PyObject *lazymanifest_getkeysiter(lazymanifest *self)
 {
 	lmIter *i = NULL;
 	lazymanifest *t = lazymanifest_copy(self);
 	if (!t) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 	i = PyObject_New(lmIter, &lazymanifestKeysIterator);
 	if (i) {
 		i->m = t;
 		i->pos = -1;
 	} else {
 		Py_DECREF(t);
 		PyErr_NoMemory();
 	}
 	return (PyObject *)i;
 }
 
 /* __getitem__ and __setitem__ support */
 
 static Py_ssize_t lazymanifest_size(lazymanifest *self)
 {
 	return self->livelines;
 }
 
 static int linecmp(const void *left, const void *right)
 {
 	return strcmp(((const line *)left)->start,
 		      ((const line *)right)->start);
 }
 
 static PyObject *lazymanifest_getitem(lazymanifest *self, PyObject *key)
 {
 	line needle;
 	line *hit;
 	if (!PyBytes_Check(key)) {
 		PyErr_Format(PyExc_TypeError,
 			     "getitem: manifest keys must be a string.");
 		return NULL;
 	}
 	needle.start = PyBytes_AsString(key);
 	hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
 		      &linecmp);
 	if (!hit || hit->deleted) {
 		PyErr_Format(PyExc_KeyError, "No such manifest entry.");
 		return NULL;
 	}
 	return hashflags(hit);
 }
 
 static int lazymanifest_delitem(lazymanifest *self, PyObject *key)
 {
 	line needle;
 	line *hit;
 	if (!PyBytes_Check(key)) {
 		PyErr_Format(PyExc_TypeError,
 			     "delitem: manifest keys must be a string.");
 		return -1;
 	}
 	needle.start = PyBytes_AsString(key);
 	hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
 		      &linecmp);
 	if (!hit || hit->deleted) {
 		PyErr_Format(PyExc_KeyError,
 			     "Tried to delete nonexistent manifest entry.");
 		return -1;
 	}
 	self->dirty = true;
 	hit->deleted = true;
 	self->livelines--;
 	return 0;
 }
 
 /* Do a binary search for the insertion point for new, creating the
  * new entry if needed. */
 static int internalsetitem(lazymanifest *self, line *new)
 {
 	int start = 0, end = self->numlines;
 	while (start < end) {
 		int pos = start + (end - start) / 2;
 		int c = linecmp(new, self->lines + pos);
 		if (c < 0)
 			end = pos;
 		else if (c > 0)
 			start = pos + 1;
 		else {
 			if (self->lines[pos].deleted)
 				self->livelines++;
 			if (self->lines[pos].from_malloc)
 				free(self->lines[pos].start);
 			start = pos;
 			goto finish;
 		}
 	}
 	/* being here means we need to do an insert */
 	if (!realloc_if_full(self)) {
 		PyErr_NoMemory();
 		return -1;
 	}
 	memmove(self->lines + start + 1, self->lines + start,
 		(self->numlines - start) * sizeof(line));
 	self->numlines++;
 	self->livelines++;
 finish:
 	self->lines[start] = *new;
 	self->dirty = true;
 	return 0;
 }
 
 static int lazymanifest_setitem(
 	lazymanifest *self, PyObject *key, PyObject *value)
 {
 	char *path;
 	Py_ssize_t plen;
 	PyObject *pyhash;
 	Py_ssize_t hlen;
 	char *hash;
 	PyObject *pyflags;
 	char *flags;
 	Py_ssize_t flen;
 	size_t dlen;
 	char *dest;
 	int i;
 	line new;
 	if (!PyBytes_Check(key)) {
 		PyErr_Format(PyExc_TypeError,
 			     "setitem: manifest keys must be a string.");
 		return -1;
 	}
 	if (!value) {
 		return lazymanifest_delitem(self, key);
 	}
 	if (!PyTuple_Check(value) || PyTuple_Size(value) != 2) {
 		PyErr_Format(PyExc_TypeError,
 			     "Manifest values must be a tuple of (node, flags).");
 		return -1;
 	}
 	if (PyBytes_AsStringAndSize(key, &path, &plen) == -1) {
 		return -1;
 	}
 
 	pyhash = PyTuple_GetItem(value, 0);
 	if (!PyBytes_Check(pyhash)) {
 		PyErr_Format(PyExc_TypeError,
 			     "node must be a 20-byte string");
 		return -1;
 	}
 	hlen = PyBytes_Size(pyhash);
 	/* Some parts of the codebase try and set 21 or 22
 	 * byte "hash" values in order to perturb things for
 	 * status. We have to preserve at least the 21st
 	 * byte. Sigh. If there's a 22nd byte, we drop it on
 	 * the floor, which works fine.
 	 */
 	if (hlen != 20 && hlen != 21 && hlen != 22) {
 		PyErr_Format(PyExc_TypeError,
 			     "node must be a 20-byte string");
 		return -1;
 	}
 	hash = PyBytes_AsString(pyhash);
 
 	pyflags = PyTuple_GetItem(value, 1);
 	if (!PyBytes_Check(pyflags) || PyBytes_Size(pyflags) > 1) {
 		PyErr_Format(PyExc_TypeError,
 			     "flags must a 0 or 1 byte string");
 		return -1;
 	}
 	if (PyBytes_AsStringAndSize(pyflags, &flags, &flen) == -1) {
 		return -1;
 	}
 	/* one null byte and one newline */
 	dlen = plen + 41 + flen + 1;
 	dest = malloc(dlen);
 	if (!dest) {
 		PyErr_NoMemory();
 		return -1;
 	}
 	memcpy(dest, path, plen + 1);
 	for (i = 0; i < 20; i++) {
 		/* Cast to unsigned, so it will not get sign-extended when promoted
 		 * to int (as is done when passing to a variadic function)
 		 */
 		sprintf(dest + plen + 1 + (i * 2), "%02x", (unsigned char)hash[i]);
 	}
 	memcpy(dest + plen + 41, flags, flen);
 	dest[plen + 41 + flen] = '\n';
 	new.start = dest;
 	new.len = dlen;
 	new.hash_suffix = '\0';
 	if (hlen > 20) {
 		new.hash_suffix = hash[20];
 	}
 	new.from_malloc = true;     /* is `start` a pointer we allocated? */
 	new.deleted = false;        /* is this entry deleted? */
 	if (internalsetitem(self, &new)) {
 		return -1;
 	}
 	return 0;
 }
 
 static PyMappingMethods lazymanifest_mapping_methods = {
 	(lenfunc)lazymanifest_size,             /* mp_length */
 	(binaryfunc)lazymanifest_getitem,       /* mp_subscript */
 	(objobjargproc)lazymanifest_setitem,    /* mp_ass_subscript */
 };
 
 /* sequence methods (important or __contains__ builds an iterator) */
 
 static int lazymanifest_contains(lazymanifest *self, PyObject *key)
 {
 	line needle;
 	line *hit;
 	if (!PyBytes_Check(key)) {
 		/* Our keys are always strings, so if the contains
 		 * check is for a non-string, just return false. */
 		return 0;
 	}
 	needle.start = PyBytes_AsString(key);
 	hit = bsearch(&needle, self->lines, self->numlines, sizeof(line),
 		      &linecmp);
 	if (!hit || hit->deleted) {
 		return 0;
 	}
 	return 1;
 }
 
 static PySequenceMethods lazymanifest_seq_meths = {
 	(lenfunc)lazymanifest_size, /* sq_length */
 	0, /* sq_concat */
 	0, /* sq_repeat */
 	0, /* sq_item */
 	0, /* sq_slice */
 	0, /* sq_ass_item */
 	0, /* sq_ass_slice */
 	(objobjproc)lazymanifest_contains, /* sq_contains */
 	0, /* sq_inplace_concat */
 	0, /* sq_inplace_repeat */
 };
 
 
 /* Other methods (copy, diff, etc) */
 static PyTypeObject lazymanifestType;
 
 /* If the manifest has changes, build the new manifest text and reindex it. */
 static int compact(lazymanifest *self)
 {
 	int i;
 	ssize_t need = 0;
 	char *data;
 	line *src, *dst;
 	PyObject *pydata;
 	if (!self->dirty)
 		return 0;
 	for (i = 0; i < self->numlines; i++) {
 		if (!self->lines[i].deleted) {
 			need += self->lines[i].len;
 		}
 	}
 	pydata = PyBytes_FromStringAndSize(NULL, need);
 	if (!pydata)
 		return -1;
 	data = PyBytes_AsString(pydata);
 	if (!data) {
 		return -1;
 	}
 	src = self->lines;
 	dst = self->lines;
 	for (i = 0; i < self->numlines; i++, src++) {
 		char *tofree = NULL;
 		if (src->from_malloc) {
 			tofree = src->start;
 		}
 		if (!src->deleted) {
 			memcpy(data, src->start, src->len);
 			*dst = *src;
 			dst->start = data;
 			dst->from_malloc = false;
 			data += dst->len;
 			dst++;
 		}
 		free(tofree);
 	}
 	Py_DECREF(self->pydata);
 	self->pydata = pydata;
 	self->numlines = self->livelines;
 	self->dirty = false;
 	return 0;
 }
 
 static PyObject *lazymanifest_text(lazymanifest *self)
 {
 	if (compact(self) != 0) {
 		PyErr_NoMemory();
 		return NULL;
 	}
 	Py_INCREF(self->pydata);
 	return self->pydata;
 }
 
 static lazymanifest *lazymanifest_copy(lazymanifest *self)
 {
 	lazymanifest *copy = NULL;
 	if (compact(self) != 0) {
 		goto nomem;
 	}
 	copy = PyObject_New(lazymanifest, &lazymanifestType);
 	if (!copy) {
 		goto nomem;
 	}
 	lazymanifest_init_early(copy);
 	copy->numlines = self->numlines;
 	copy->livelines = self->livelines;
 	copy->dirty = false;
 	copy->lines = malloc(self->maxlines *sizeof(line));
 	if (!copy->lines) {
 		goto nomem;
 	}
 	memcpy(copy->lines, self->lines, self->numlines * sizeof(line));
 	copy->maxlines = self->maxlines;
 	copy->pydata = self->pydata;
 	Py_INCREF(copy->pydata);
 	return copy;
 nomem:
 	PyErr_NoMemory();
 	Py_XDECREF(copy);
 	return NULL;
 }
 
 static lazymanifest *lazymanifest_filtercopy(
 	lazymanifest *self, PyObject *matchfn)
 {
 	lazymanifest *copy = NULL;
 	int i;
 	if (!PyCallable_Check(matchfn)) {
 		PyErr_SetString(PyExc_TypeError, "matchfn must be callable");
 		return NULL;
 	}
 	/* compact ourselves first to avoid double-frees later when we
 	 * compact tmp so that it doesn't have random pointers to our
 	 * underlying from_malloc-data (self->pydata is safe) */
 	if (compact(self) != 0) {
 		goto nomem;
 	}
 	copy = PyObject_New(lazymanifest, &lazymanifestType);
 	if (!copy) {
 		goto nomem;
 	}
 	lazymanifest_init_early(copy);
 	copy->dirty = true;
 	copy->lines = malloc(self->maxlines * sizeof(line));
 	if (!copy->lines) {
 		goto nomem;
 	}
 	copy->maxlines = self->maxlines;
 	copy->numlines = 0;
 	copy->pydata = self->pydata;
 	Py_INCREF(copy->pydata);
 	for (i = 0; i < self->numlines; i++) {
 		PyObject *arglist = NULL, *result = NULL;
 		arglist = Py_BuildValue(PY23("(s)", "(y)"),
 					self->lines[i].start);
 		if (!arglist) {
 			goto bail;
 		}
 		result = PyObject_CallObject(matchfn, arglist);
 		Py_DECREF(arglist);
 		/* if the callback raised an exception, just let it
 		 * through and give up */
 		if (!result) {
 			goto bail;
 		}
 		if (PyObject_IsTrue(result)) {
 			assert(!(self->lines[i].from_malloc));
 			copy->lines[copy->numlines++] = self->lines[i];
 		}
 		Py_DECREF(result);
 	}
 	copy->livelines = copy->numlines;
 	return copy;
 nomem:
 	PyErr_NoMemory();
 bail:
 	Py_XDECREF(copy);
 	return NULL;
 }
 
 static PyObject *lazymanifest_diff(lazymanifest *self, PyObject *args)
 {
 	lazymanifest *other;
 	PyObject *pyclean = NULL;
 	bool listclean;
 	PyObject *emptyTup = NULL, *ret = NULL;
 	PyObject *es;
 	int sneedle = 0, oneedle = 0;
 	if (!PyArg_ParseTuple(args, "O!|O", &lazymanifestType, &other, &pyclean)) {
 		return NULL;
 	}
 	listclean = (!pyclean) ? false : PyObject_IsTrue(pyclean);
 	es = PyBytes_FromString("");
 	if (!es) {
 		goto nomem;
 	}
 	emptyTup = PyTuple_Pack(2, Py_None, es);
 	Py_DECREF(es);
 	if (!emptyTup) {
 		goto nomem;
 	}
 	ret = PyDict_New();
 	if (!ret) {
 		goto nomem;
 	}
 	while (sneedle != self->numlines || oneedle != other->numlines) {
 		line *left = self->lines + sneedle;
 		line *right = other->lines + oneedle;
 		int result;
 		PyObject *key;
 		PyObject *outer;
 		/* If we're looking at a deleted entry and it's not
 		 * the end of the manifest, just skip it. */
 		if (sneedle < self->numlines && left->deleted) {
 			sneedle++;
 			continue;
 		}
 		if (oneedle < other->numlines && right->deleted) {
 			oneedle++;
 			continue;
 		}
 		/* if we're at the end of either manifest, then we
 		 * know the remaining items are adds so we can skip
 		 * the strcmp. */
 		if (sneedle == self->numlines) {
 			result = 1;
 		} else if (oneedle == other->numlines) {
 			result = -1;
 		} else {
 			result = linecmp(left, right);
 		}
 		key = result <= 0 ?
 			PyBytes_FromString(left->start) :
 			PyBytes_FromString(right->start);
 		if (!key)
 			goto nomem;
 		if (result < 0) {
 			PyObject *l = hashflags(left);
 			if (!l) {
 				goto nomem;
 			}
 			outer = PyTuple_Pack(2, l, emptyTup);
 			Py_DECREF(l);
 			if (!outer) {
 				goto nomem;
 			}
 			PyDict_SetItem(ret, key, outer);
 			Py_DECREF(outer);
 			sneedle++;
 		} else if (result > 0) {
 			PyObject *r = hashflags(right);
 			if (!r) {
 				goto nomem;
 			}
 			outer = PyTuple_Pack(2, emptyTup, r);
 			Py_DECREF(r);
 			if (!outer) {
 				goto nomem;
 			}
 			PyDict_SetItem(ret, key, outer);
 			Py_DECREF(outer);
 			oneedle++;
 		} else {
 			/* file exists in both manifests */
 			if (left->len != right->len
 			    || memcmp(left->start, right->start, left->len)
 			    || left->hash_suffix != right->hash_suffix) {
 				PyObject *l = hashflags(left);
 				PyObject *r;
 				if (!l) {
 					goto nomem;
 				}
 				r = hashflags(right);
 				if (!r) {
 					Py_DECREF(l);
 					goto nomem;
 				}
 				outer = PyTuple_Pack(2, l, r);
 				Py_DECREF(l);
 				Py_DECREF(r);
 				if (!outer) {
 					goto nomem;
 				}
 				PyDict_SetItem(ret, key, outer);
 				Py_DECREF(outer);
 			} else if (listclean) {
 				PyDict_SetItem(ret, key, Py_None);
 			}
 			sneedle++;
 			oneedle++;
 		}
 		Py_DECREF(key);
 	}
 	Py_DECREF(emptyTup);
 	return ret;
 nomem:
 	PyErr_NoMemory();
 	Py_XDECREF(ret);
 	Py_XDECREF(emptyTup);
 	return NULL;
 }
 
 static PyMethodDef lazymanifest_methods[] = {
 	{"iterkeys", (PyCFunction)lazymanifest_getkeysiter, METH_NOARGS,
 	 "Iterate over file names in this lazymanifest."},
 	{"iterentries", (PyCFunction)lazymanifest_getentriesiter, METH_NOARGS,
 	 "Iterate over (path, nodeid, flags) tuples in this lazymanifest."},
 	{"copy", (PyCFunction)lazymanifest_copy, METH_NOARGS,
 	 "Make a copy of this lazymanifest."},
 	{"filtercopy", (PyCFunction)lazymanifest_filtercopy, METH_O,
 	 "Make a copy of this manifest filtered by matchfn."},
 	{"diff", (PyCFunction)lazymanifest_diff, METH_VARARGS,
 	 "Compare this lazymanifest to another one."},
 	{"text", (PyCFunction)lazymanifest_text, METH_NOARGS,
 	 "Encode this manifest to text."},
 	{NULL},
 };
 
 #ifdef IS_PY3K
 #define LAZYMANIFEST_TPFLAGS Py_TPFLAGS_DEFAULT
 #else
 #define LAZYMANIFEST_TPFLAGS Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_SEQUENCE_IN
 #endif
 
 static PyTypeObject lazymanifestType = {
 	PyVarObject_HEAD_INIT(NULL, 0) /* header */
 	"parsers.lazymanifest",                           /* tp_name */
 	sizeof(lazymanifest),                             /* tp_basicsize */
 	0,                                                /* tp_itemsize */
 	(destructor)lazymanifest_dealloc,                 /* tp_dealloc */
 	0,                                                /* tp_print */
 	0,                                                /* tp_getattr */
 	0,                                                /* tp_setattr */
 	0,                                                /* tp_compare */
 	0,                                                /* tp_repr */
 	0,                                                /* tp_as_number */
 	&lazymanifest_seq_meths,                          /* tp_as_sequence */
 	&lazymanifest_mapping_methods,                    /* tp_as_mapping */
 	0,                                                /* tp_hash */
 	0,                                                /* tp_call */
 	0,                                                /* tp_str */
 	0,                                                /* tp_getattro */
 	0,                                                /* tp_setattro */
 	0,                                                /* tp_as_buffer */
 	LAZYMANIFEST_TPFLAGS,                             /* tp_flags */
 	"TODO(augie)",                                    /* tp_doc */
 	0,                                                /* tp_traverse */
 	0,                                                /* tp_clear */
 	0,                                                /* tp_richcompare */
 	0,                                             /* tp_weaklistoffset */
 	(getiterfunc)lazymanifest_getkeysiter,                /* tp_iter */
 	0,                                                /* tp_iternext */
 	lazymanifest_methods,                             /* tp_methods */
 	0,                                                /* tp_members */
 	0,                                                /* tp_getset */
 	0,                                                /* tp_base */
 	0,                                                /* tp_dict */
 	0,                                                /* tp_descr_get */
 	0,                                                /* tp_descr_set */
 	0,                                                /* tp_dictoffset */
 	(initproc)lazymanifest_init,                      /* tp_init */
 	0,                                                /* tp_alloc */
 };
 
 void manifest_module_init(PyObject * mod)
 {
 	lazymanifestType.tp_new = PyType_GenericNew;
 	if (PyType_Ready(&lazymanifestType) < 0)
 		return;
 	Py_INCREF(&lazymanifestType);
 
 	PyModule_AddObject(mod, "lazymanifest",
 			   (PyObject *)&lazymanifestType);
 }

tests/test-manifest.py

@@ -1,422 +1,446
 from __future__ import absolute_import
 
 import binascii
 import itertools
 import silenttestrunner
 import unittest
+import zlib
 
 from mercurial import (
     manifest as manifestmod,
     match as matchmod,
 )
 
 EMTPY_MANIFEST = b''
 
 HASH_1 = b'1' * 40
 BIN_HASH_1 = binascii.unhexlify(HASH_1)
 HASH_2 = b'f' * 40
 BIN_HASH_2 = binascii.unhexlify(HASH_2)
 HASH_3 = b'1234567890abcdef0987654321deadbeef0fcafe'
 BIN_HASH_3 = binascii.unhexlify(HASH_3)
 A_SHORT_MANIFEST = (
     b'bar/baz/qux.py\0%(hash2)s%(flag2)s\n'
     b'foo\0%(hash1)s%(flag1)s\n'
     ) % {b'hash1': HASH_1,
          b'flag1': b'',
          b'hash2': HASH_2,
          b'flag2': b'l',
          }
 
 A_DEEPER_MANIFEST = (
     b'a/b/c/bar.py\0%(hash3)s%(flag1)s\n'
     b'a/b/c/bar.txt\0%(hash1)s%(flag1)s\n'
     b'a/b/c/foo.py\0%(hash3)s%(flag1)s\n'
     b'a/b/c/foo.txt\0%(hash2)s%(flag2)s\n'
     b'a/b/d/baz.py\0%(hash3)s%(flag1)s\n'
     b'a/b/d/qux.py\0%(hash1)s%(flag2)s\n'
     b'a/b/d/ten.txt\0%(hash3)s%(flag2)s\n'
     b'a/b/dog.py\0%(hash3)s%(flag1)s\n'
     b'a/b/fish.py\0%(hash2)s%(flag1)s\n'
     b'a/c/london.py\0%(hash3)s%(flag2)s\n'
     b'a/c/paper.txt\0%(hash2)s%(flag2)s\n'
     b'a/c/paris.py\0%(hash2)s%(flag1)s\n'
     b'a/d/apple.py\0%(hash3)s%(flag1)s\n'
     b'a/d/pizza.py\0%(hash3)s%(flag2)s\n'
     b'a/green.py\0%(hash1)s%(flag2)s\n'
     b'a/purple.py\0%(hash2)s%(flag1)s\n'
     b'app.py\0%(hash3)s%(flag1)s\n'
     b'readme.txt\0%(hash2)s%(flag1)s\n'
     ) % {b'hash1': HASH_1,
          b'flag1': b'',
          b'hash2': HASH_2,
          b'flag2': b'l',
          b'hash3': HASH_3,
          }
 
 HUGE_MANIFEST_ENTRIES = 200001
 
 izip = getattr(itertools, 'izip', zip)
 if 'xrange' not in globals():
     xrange = range
 
 A_HUGE_MANIFEST = b''.join(sorted(
     b'file%d\0%s%s\n' % (i, h, f) for i, h, f in
     izip(xrange(200001),
          itertools.cycle((HASH_1, HASH_2)),
          itertools.cycle((b'', b'x', b'l')))))
 
 class basemanifesttests(object):
     def parsemanifest(self, text):
         raise NotImplementedError('parsemanifest not implemented by test case')
 
     def testEmptyManifest(self):
         m = self.parsemanifest(EMTPY_MANIFEST)
         self.assertEqual(0, len(m))
         self.assertEqual([], list(m))
 
     def testManifest(self):
         m = self.parsemanifest(A_SHORT_MANIFEST)
         self.assertEqual([b'bar/baz/qux.py', b'foo'], list(m))
         self.assertEqual(BIN_HASH_2, m[b'bar/baz/qux.py'])
         self.assertEqual(b'l', m.flags(b'bar/baz/qux.py'))
         self.assertEqual(BIN_HASH_1, m[b'foo'])
         self.assertEqual(b'', m.flags(b'foo'))
         with self.assertRaises(KeyError):
             m[b'wat']
 
     def testSetItem(self):
         want = BIN_HASH_1
 
         m = self.parsemanifest(EMTPY_MANIFEST)
         m[b'a'] = want
         self.assertIn(b'a', m)
         self.assertEqual(want, m[b'a'])
         self.assertEqual(b'a\0' + HASH_1 + b'\n', m.text())
 
         m = self.parsemanifest(A_SHORT_MANIFEST)
         m[b'a'] = want
         self.assertEqual(want, m[b'a'])
         self.assertEqual(b'a\0' + HASH_1 + b'\n' + A_SHORT_MANIFEST,
                          m.text())
 
     def testSetFlag(self):
         want = b'x'
 
         m = self.parsemanifest(EMTPY_MANIFEST)
         # first add a file; a file-less flag makes no sense
         m[b'a'] = BIN_HASH_1
         m.setflag(b'a', want)
         self.assertEqual(want, m.flags(b'a'))
         self.assertEqual(b'a\0' + HASH_1 + want + b'\n', m.text())
 
         m = self.parsemanifest(A_SHORT_MANIFEST)
         # first add a file; a file-less flag makes no sense
         m[b'a'] = BIN_HASH_1
         m.setflag(b'a', want)
         self.assertEqual(want, m.flags(b'a'))
         self.assertEqual(b'a\0' + HASH_1 + want + b'\n' + A_SHORT_MANIFEST,
                          m.text())
 
     def testCopy(self):
         m = self.parsemanifest(A_SHORT_MANIFEST)
         m[b'a'] = BIN_HASH_1
         m2 = m.copy()
         del m
         del m2 # make sure we don't double free() anything
 
     def testCompaction(self):
         unhex = binascii.unhexlify
         h1, h2 = unhex(HASH_1), unhex(HASH_2)
         m = self.parsemanifest(A_SHORT_MANIFEST)
         m[b'alpha'] = h1
         m[b'beta'] = h2
         del m[b'foo']
         want = b'alpha\0%s\nbar/baz/qux.py\0%sl\nbeta\0%s\n' % (
             HASH_1, HASH_2, HASH_2)
         self.assertEqual(want, m.text())
         self.assertEqual(3, len(m))
         self.assertEqual([b'alpha', b'bar/baz/qux.py', b'beta'], list(m))
         self.assertEqual(h1, m[b'alpha'])
         self.assertEqual(h2, m[b'bar/baz/qux.py'])
         self.assertEqual(h2, m[b'beta'])
         self.assertEqual(b'', m.flags(b'alpha'))
         self.assertEqual(b'l', m.flags(b'bar/baz/qux.py'))
         self.assertEqual(b'', m.flags(b'beta'))
         with self.assertRaises(KeyError):
             m[b'foo']
 
     def testSetGetNodeSuffix(self):
         clean = self.parsemanifest(A_SHORT_MANIFEST)
         m = self.parsemanifest(A_SHORT_MANIFEST)
         h = m[b'foo']
         f = m.flags(b'foo')
         want = h + b'a'
         # Merge code wants to set 21-byte fake hashes at times
         m[b'foo'] = want
         self.assertEqual(want, m[b'foo'])
         self.assertEqual([(b'bar/baz/qux.py', BIN_HASH_2),
                           (b'foo', BIN_HASH_1 + b'a')],
                          list(m.items()))
         # Sometimes it even tries a 22-byte fake hash, but we can
         # return 21 and it'll work out
         m[b'foo'] = want + b'+'
         self.assertEqual(want, m[b'foo'])
         # make sure the suffix survives a copy
         match = matchmod.match(b'', b'', [b're:foo'])
         m2 = m.matches(match)
         self.assertEqual(want, m2[b'foo'])
         self.assertEqual(1, len(m2))
         m2 = m.copy()
         self.assertEqual(want, m2[b'foo'])
         # suffix with iteration
         self.assertEqual([(b'bar/baz/qux.py', BIN_HASH_2),
                           (b'foo', want)],
                          list(m.items()))
 
         # shows up in diff
         self.assertEqual({b'foo': ((want, f), (h, b''))}, m.diff(clean))
         self.assertEqual({b'foo': ((h, b''), (want, f))}, clean.diff(m))
 
     def testMatchException(self):
         m = self.parsemanifest(A_SHORT_MANIFEST)
         match = matchmod.match(b'', b'', [b're:.*'])
         def filt(path):
             if path == b'foo':
                 assert False
             return True
         match.matchfn = filt
         with self.assertRaises(AssertionError):
             m.matches(match)
 
     def testRemoveItem(self):
         m = self.parsemanifest(A_SHORT_MANIFEST)
         del m[b'foo']
         with self.assertRaises(KeyError):
             m[b'foo']
         self.assertEqual(1, len(m))
         self.assertEqual(1, len(list(m)))
         # now restore and make sure everything works right
         m[b'foo'] = b'a' * 20
         self.assertEqual(2, len(m))
         self.assertEqual(2, len(list(m)))
 
     def testManifestDiff(self):
         MISSING = (None, b'')
         addl = b'z-only-in-left\0' + HASH_1 + b'\n'
         addr = b'z-only-in-right\0' + HASH_2 + b'x\n'
         left = self.parsemanifest(
             A_SHORT_MANIFEST.replace(HASH_1, HASH_3 + b'x') + addl)
         right = self.parsemanifest(A_SHORT_MANIFEST + addr)
         want = {
             b'foo': ((BIN_HASH_3, b'x'),
                      (BIN_HASH_1, b'')),
             b'z-only-in-left': ((BIN_HASH_1, b''), MISSING),
             b'z-only-in-right': (MISSING, (BIN_HASH_2, b'x')),
             }
         self.assertEqual(want, left.diff(right))
 
         want = {
             b'bar/baz/qux.py': (MISSING, (BIN_HASH_2, b'l')),
             b'foo': (MISSING, (BIN_HASH_3, b'x')),
             b'z-only-in-left': (MISSING, (BIN_HASH_1, b'')),
             }
         self.assertEqual(want, self.parsemanifest(EMTPY_MANIFEST).diff(left))
 
         want = {
             b'bar/baz/qux.py': ((BIN_HASH_2, b'l'), MISSING),
             b'foo': ((BIN_HASH_3, b'x'), MISSING),
             b'z-only-in-left': ((BIN_HASH_1, b''), MISSING),
             }
         self.assertEqual(want, left.diff(self.parsemanifest(EMTPY_MANIFEST)))
         copy = right.copy()
         del copy[b'z-only-in-right']
         del right[b'foo']
         want = {
             b'foo': (MISSING, (BIN_HASH_1, b'')),
             b'z-only-in-right': ((BIN_HASH_2, b'x'), MISSING),
             }
         self.assertEqual(want, right.diff(copy))
 
         short = self.parsemanifest(A_SHORT_MANIFEST)
         pruned = short.copy()
         del pruned[b'foo']
         want = {
             b'foo': ((BIN_HASH_1, b''), MISSING),
             }
         self.assertEqual(want, short.diff(pruned))
         want = {
             b'foo': (MISSING, (BIN_HASH_1, b'')),
             }
         self.assertEqual(want, pruned.diff(short))
         want = {
             b'bar/baz/qux.py': None,
             b'foo': (MISSING, (BIN_HASH_1, b'')),
             }
         self.assertEqual(want, pruned.diff(short, clean=True))
 
     def testReversedLines(self):
         backwards = b''.join(
             l + b'\n' for l in reversed(A_SHORT_MANIFEST.split(b'\n')) if l)
         try:
             self.parsemanifest(backwards)
             self.fail('Should have raised ValueError')
         except ValueError as v:
             self.assertIn('Manifest lines not in sorted order.', str(v))
 
     def testNoTerminalNewline(self):
         try:
             self.parsemanifest(A_SHORT_MANIFEST + b'wat')
             self.fail('Should have raised ValueError')
         except ValueError as v:
             self.assertIn('Manifest did not end in a newline.', str(v))
 
     def testNoNewLineAtAll(self):
         try:
             self.parsemanifest(b'wat')
             self.fail('Should have raised ValueError')
         except ValueError as v:
             self.assertIn('Manifest did not end in a newline.', str(v))
 
     def testHugeManifest(self):
         m = self.parsemanifest(A_HUGE_MANIFEST)
         self.assertEqual(HUGE_MANIFEST_ENTRIES, len(m))
         self.assertEqual(len(m), len(list(m)))
 
     def testMatchesMetadata(self):
         '''Tests matches() for a few specific files to make sure that both
         the set of files as well as their flags and nodeids are correct in
         the resulting manifest.'''
         m = self.parsemanifest(A_HUGE_MANIFEST)
 
         match = matchmod.match(b'/', b'',
                 [b'file1', b'file200', b'file300'], exact=True)
         m2 = m.matches(match)
 
         w = (b'file1\0%sx\n'
              b'file200\0%sl\n'
              b'file300\0%s\n') % (HASH_2, HASH_1, HASH_1)
         self.assertEqual(w, m2.text())
 
     def testMatchesNonexistentFile(self):
         '''Tests matches() for a small set of specific files, including one
         nonexistent file to make sure in only matches against existing files.
         '''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'',
                 [b'a/b/c/bar.txt', b'a/b/d/qux.py',
                  b'readme.txt', b'nonexistent'],
                 exact=True)
         m2 = m.matches(match)
 
         self.assertEqual(
                 [b'a/b/c/bar.txt', b'a/b/d/qux.py', b'readme.txt'],
                 m2.keys())
 
     def testMatchesNonexistentDirectory(self):
         '''Tests matches() for a relpath match on a directory that doesn't
         actually exist.'''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'', [b'a/f'], default=b'relpath')
         m2 = m.matches(match)
 
         self.assertEqual([], m2.keys())
 
     def testMatchesExactLarge(self):
         '''Tests matches() for files matching a large list of exact files.
         '''
         m = self.parsemanifest(A_HUGE_MANIFEST)
 
         flist = m.keys()[80:300]
         match = matchmod.match(b'/', b'', flist, exact=True)
         m2 = m.matches(match)
 
         self.assertEqual(flist, m2.keys())
 
     def testMatchesFull(self):
         '''Tests matches() for what should be a full match.'''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'', [b''])
         m2 = m.matches(match)
 
         self.assertEqual(m.keys(), m2.keys())
 
     def testMatchesDirectory(self):
         '''Tests matches() on a relpath match on a directory, which should
         match against all files within said directory.'''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'', [b'a/b'], default=b'relpath')
         m2 = m.matches(match)
 
         self.assertEqual([
             b'a/b/c/bar.py', b'a/b/c/bar.txt', b'a/b/c/foo.py',
             b'a/b/c/foo.txt',
             b'a/b/d/baz.py', b'a/b/d/qux.py', b'a/b/d/ten.txt', b'a/b/dog.py',
             b'a/b/fish.py'], m2.keys())
 
     def testMatchesExactPath(self):
         '''Tests matches() on an exact match on a directory, which should
         result in an empty manifest because you can't perform an exact match
         against a directory.'''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'', [b'a/b'], exact=True)
         m2 = m.matches(match)
 
         self.assertEqual([], m2.keys())
 
     def testMatchesCwd(self):
         '''Tests matches() on a relpath match with the current directory ('.')
         when not in the root directory.'''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'a/b', [b'.'], default=b'relpath')
         m2 = m.matches(match)
 
         self.assertEqual([
             b'a/b/c/bar.py', b'a/b/c/bar.txt', b'a/b/c/foo.py',
             b'a/b/c/foo.txt', b'a/b/d/baz.py', b'a/b/d/qux.py',
             b'a/b/d/ten.txt', b'a/b/dog.py', b'a/b/fish.py'], m2.keys())
 
     def testMatchesWithPattern(self):
         '''Tests matches() for files matching a pattern that reside
         deeper than the specified directory.'''
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         match = matchmod.match(b'/', b'', [b'a/b/*/*.txt'])
         m2 = m.matches(match)
 
         self.assertEqual(
                 [b'a/b/c/bar.txt', b'a/b/c/foo.txt', b'a/b/d/ten.txt'],
                 m2.keys())
 
 class testmanifestdict(unittest.TestCase, basemanifesttests):
     def parsemanifest(self, text):
         return manifestmod.manifestdict(text)
 
+    def testObviouslyBogusManifest(self):
+        # This is a 163k manifest that came from oss-fuzz. It was a
+        # timeout there, but when run normally it doesn't seem to
+        # present any particular slowness.
+        data = zlib.decompress(
+            'x\x9c\xed\xce;\n\x83\x00\x10\x04\xd0\x8deNa\x93~\xf1\x03\xc9q\xf4'
+            '\x14\xeaU\xbdB\xda\xd4\xe6Cj\xc1FA\xde+\x86\xe9f\xa2\xfci\xbb\xfb'
+            '\xa3\xef\xea\xba\xca\x7fk\x86q\x9a\xc6\xc8\xcc&\xb3\xcf\xf8\xb8|#'
+            '\x8a9\x00\xd8\xe6v\xf4\x01N\xe1\n\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+            '\x00\x00\xc0\x8aey\x1d}\x01\xd8\xe0\xb9\xf3\xde\x1b\xcf\x17'
+            '\xac\xbe')
+        with self.assertRaises(ValueError):
+            self.parsemanifest(data)
+
 class testtreemanifest(unittest.TestCase, basemanifesttests):
     def parsemanifest(self, text):
         return manifestmod.treemanifest(b'', text)
 
     def testWalkSubtrees(self):
         m = self.parsemanifest(A_DEEPER_MANIFEST)
 
         dirs = [s._dir for s in m.walksubtrees()]
         self.assertEqual(
             sorted([
                 b'', b'a/', b'a/c/', b'a/d/', b'a/b/', b'a/b/c/', b'a/b/d/']),
             sorted(dirs)
         )
 
         match = matchmod.match(b'/', b'', [b'path:a/b/'])
         dirs = [s._dir for s in m.walksubtrees(matcher=match)]
         self.assertEqual(
             sorted([b'a/b/', b'a/b/c/', b'a/b/d/']),
             sorted(dirs)
         )
 
 if __name__ == '__main__':
     silenttestrunner.main(__name__)