bdiff: early pruning of common prefix before doing expensive computations

It seems quite common that files don't change completely. New lines are often
pretty much appended, and modifications will often only change a small section
of the file which on average will be in the middle.

There can thus be a big win by pruning a common prefix before starting the more
expensive search for longest common substrings.

Worst case, it will scan through a long sequence of similar bytes without
encountering a newline. Splitlines will then have to do the same again ...
twice for each side. If similar lines are found, splitlines will save the
double iteration and hashing of the lines ... plus there will be less lines to
find common substrings in.

This change might in some cases make the algorith pick shorter or less optimal
common substrings. We can't have the cake and eat it.

This make hg --time bundle --base null -r 4.0 go from 14.5 to 15 s - a 3%
increase.

On mozilla-unified:
perfbdiff -m 3041e4d59df2
! wall 0.053088 comb 0.060000 user 0.060000 sys 0.000000 (best of 100) to
! wall 0.024618 comb 0.020000 user 0.020000 sys 0.000000 (best of 116)
perfbdiff 0e9928989e9c --alldata --count 10
! wall 0.702075 comb 0.700000 user 0.700000 sys 0.000000 (best of 15) to
! wall 0.579235 comb 0.580000 user 0.580000 sys 0.000000 (best of 18)

mercurial/bdiff_module.c

@@ -1,204 +1,213 b''
 /*
  bdiff.c - efficient binary diff extension for Mercurial
 
  Copyright 2005, 2006 Matt Mackall <[e-mail]>
 
  This software may be used and distributed according to the terms of
  the GNU General Public License, incorporated herein by reference.
 
  Based roughly on Python difflib
 */
 
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
 
 #include "bdiff.h"
 #include "bitmanipulation.h"
 #include "util.h"
 
 
 static PyObject *blocks(PyObject *self, PyObject *args)
 {
 	PyObject *sa, *sb, *rl = NULL, *m;
 	struct bdiff_line *a, *b;
 	struct bdiff_hunk l, *h;
 	int an, bn, count, pos = 0;
 
 	l.next = NULL;
 
 	if (!PyArg_ParseTuple(args, "SS:bdiff", &sa, &sb))
 		return NULL;
 
 	an = bdiff_splitlines(PyBytes_AsString(sa), PyBytes_Size(sa), &a);
 	bn = bdiff_splitlines(PyBytes_AsString(sb), PyBytes_Size(sb), &b);
 
 	if (!a || !b)
 		goto nomem;
 
 	count = bdiff_diff(a, an, b, bn, &l);
 	if (count < 0)
 		goto nomem;
 
 	rl = PyList_New(count);
 	if (!rl)
 		goto nomem;
 
 	for (h = l.next; h; h = h->next) {
 		m = Py_BuildValue("iiii", h->a1, h->a2, h->b1, h->b2);
 		PyList_SetItem(rl, pos, m);
 		pos++;
 	}
 
 nomem:
 	free(a);
 	free(b);
 	bdiff_freehunks(l.next);
 	return rl ? rl : PyErr_NoMemory();
 }
 
 static PyObject *bdiff(PyObject *self, PyObject *args)
 {
-	char *sa, *sb, *rb;
+	char *sa, *sb, *rb, *ia, *ib;
 	PyObject *result = NULL;
 	struct bdiff_line *al, *bl;
 	struct bdiff_hunk l, *h;
 	int an, bn, count;
-	Py_ssize_t len = 0, la, lb;
+	Py_ssize_t len = 0, la, lb, li = 0, lcommon = 0, lmax;
 	PyThreadState *_save;
 
 	l.next = NULL;
 
 	if (!PyArg_ParseTuple(args, "s#s#:bdiff", &sa, &la, &sb, &lb))
 		return NULL;
 
 	if (la > UINT_MAX || lb > UINT_MAX) {
 		PyErr_SetString(PyExc_ValueError, "bdiff inputs too large");
 		return NULL;
 	}
 
 	_save = PyEval_SaveThread();
-	an = bdiff_splitlines(sa, la, &al);
-	bn = bdiff_splitlines(sb, lb, &bl);
+
+	lmax = la > lb ? lb : la;
+	for (ia = sa, ib = sb;
+	     li < lmax && *ia == *ib;
+	     ++li, ++ia, ++ib)
+		if (*ia == '\n')
+			lcommon = li + 1;
+	/* we can almost add: if (li == lmax) lcommon = li; */
+
+	an = bdiff_splitlines(sa + lcommon, la - lcommon, &al);
+	bn = bdiff_splitlines(sb + lcommon, lb - lcommon, &bl);
 	if (!al || !bl)
 		goto nomem;
 
 	count = bdiff_diff(al, an, bl, bn, &l);
 	if (count < 0)
 		goto nomem;
 
 	/* calculate length of output */
 	la = lb = 0;
 	for (h = l.next; h; h = h->next) {
 		if (h->a1 != la || h->b1 != lb)
 			len += 12 + bl[h->b1].l - bl[lb].l;
 		la = h->a2;
 		lb = h->b2;
 	}
 	PyEval_RestoreThread(_save);
 	_save = NULL;
 
 	result = PyBytes_FromStringAndSize(NULL, len);
 
 	if (!result)
 		goto nomem;
 
 	/* build binary patch */
 	rb = PyBytes_AsString(result);
 	la = lb = 0;
 
 	for (h = l.next; h; h = h->next) {
 		if (h->a1 != la || h->b1 != lb) {
 			len = bl[h->b1].l - bl[lb].l;
-			putbe32((uint32_t)(al[la].l - al->l), rb);
-			putbe32((uint32_t)(al[h->a1].l - al->l), rb + 4);
+			putbe32((uint32_t)(al[la].l + lcommon - al->l), rb);
+			putbe32((uint32_t)(al[h->a1].l + lcommon - al->l), rb + 4);
 			putbe32((uint32_t)len, rb + 8);
 			memcpy(rb + 12, bl[lb].l, len);
 			rb += 12 + len;
 		}
 		la = h->a2;
 		lb = h->b2;
 	}
 
 nomem:
 	if (_save)
 		PyEval_RestoreThread(_save);
 	free(al);
 	free(bl);
 	bdiff_freehunks(l.next);
 	return result ? result : PyErr_NoMemory();
 }
 
 /*
  * If allws != 0, remove all whitespace (' ', \t and \r). Otherwise,
  * reduce whitespace sequences to a single space and trim remaining whitespace
  * from end of lines.
  */
 static PyObject *fixws(PyObject *self, PyObject *args)
 {
 	PyObject *s, *result = NULL;
 	char allws, c;
 	const char *r;
 	Py_ssize_t i, rlen, wlen = 0;
 	char *w;
 
 	if (!PyArg_ParseTuple(args, "Sb:fixws", &s, &allws))
 		return NULL;
 	r = PyBytes_AsString(s);
 	rlen = PyBytes_Size(s);
 
 	w = (char *)malloc(rlen ? rlen : 1);
 	if (!w)
 		goto nomem;
 
 	for (i = 0; i != rlen; i++) {
 		c = r[i];
 		if (c == ' ' || c == '\t' || c == '\r') {
 			if (!allws && (wlen == 0 || w[wlen - 1] != ' '))
 				w[wlen++] = ' ';
 		} else if (c == '\n' && !allws
 			  && wlen > 0 && w[wlen - 1] == ' ') {
 			w[wlen - 1] = '\n';
 		} else {
 			w[wlen++] = c;
 		}
 	}
 
 	result = PyBytes_FromStringAndSize(w, wlen);
 
 nomem:
 	free(w);
 	return result ? result : PyErr_NoMemory();
 }
 
 
 static char mdiff_doc[] = "Efficient binary diff.";
 
 static PyMethodDef methods[] = {
 	{"bdiff", bdiff, METH_VARARGS, "calculate a binary diff\n"},
 	{"blocks", blocks, METH_VARARGS, "find a list of matching lines\n"},
 	{"fixws", fixws, METH_VARARGS, "normalize diff whitespaces\n"},
 	{NULL, NULL}
 };
 
 #ifdef IS_PY3K
 static struct PyModuleDef bdiff_module = {
 	PyModuleDef_HEAD_INIT,
 	"bdiff",
 	mdiff_doc,
 	-1,
 	methods
 };
 
 PyMODINIT_FUNC PyInit_bdiff(void)
 {
 	return PyModule_Create(&bdiff_module);
 }
 #else
 PyMODINIT_FUNC initbdiff(void)
 {
 	Py_InitModule3("bdiff", methods, mdiff_doc);
 }
 #endif

tests/test-bdiff.py.out

@@ -1,82 +1,81 b''
 test 'a\nc\n\n\n\n' 'a\nb\n\n\n'
 test 'a\nb\nc\n' 'a\nc\n'
 test '' ''
 test 'a\nb\nc' 'a\nb\nc'
 test 'a\nb\nc\nd\n' 'a\nd\n'
 test 'a\nb\nc\nd\n' 'a\nc\ne\n'
 test 'a\nb\nc\n' 'a\nc\n'
 test 'a\n' 'c\na\nb\n'
 test 'a\n' ''
 test 'a\n' 'b\nc\n'
 test 'a\n' 'c\na\n'
 test '' 'adjfkjdjksdhfksj'
 test '' 'ab'
 test '' 'abc'
 test 'a' 'a'
 test 'ab' 'ab'
 test 'abc' 'abc'
 test 'a\n' 'a\n'
 test 'a\nb' 'a\nb'
 showdiff(
   'x\n\nx\n\nx\n\nx\n\nz\n',
   'x\n\nx\n\ny\n\nx\n\nx\n\nz\n'):
  'x\n\nx\n\n'
  6 6 '' -> 'y\n\n'
  'x\n\nx\n\nz\n'
 showdiff(
   'x\n\nx\n\nx\n\nx\n\nz\n',
   'x\n\nx\n\ny\n\nx\n\ny\n\nx\n\nz\n'):
  'x\n\nx\n\n'
  6 6 '' -> 'y\n\n'
  'x\n\n'
  9 9 '' -> 'y\n\n'
  'x\n\nz\n'
 showdiff(
   'a\nb\nb\nb\nc\n.\nd\ne\n.\nf\n',
   'a\nb\nb\na\nb\nb\nb\nc\n.\nb\nc\n.\nd\ne\nf\n'):
- 0 0 '' -> 'a\nb\nb\n'
- 'a\nb\nb\nb\nc\n.\n'
- 12 12 '' -> 'b\nc\n.\n'
- 'd\ne\n'
+ 'a\nb\nb\n'
+ 6 6 '' -> 'a\nb\nb\nb\nc\n.\n'
+ 'b\nc\n.\nd\ne\n'
  16 18 '.\n' -> ''
  'f\n'
 done
 done
 Nice diff for a trivial change:
 showdiff(
   '<0\n-\n<1\n-\n<2\n-\n<3\n-\n<4\n-\n',
   '>0\n-\n>1\n-\n>2\n-\n>3\n-\n>4\n-\n'):
  0 3 '<0\n' -> '>0\n'
  '-\n'
  5 8 '<1\n' -> '>1\n'
  '-\n'
  10 13 '<2\n' -> '>2\n'
  '-\n'
  15 18 '<3\n' -> '>3\n'
  '-\n'
  20 23 '<4\n' -> '>4\n'
  '-\n'
 Diff 1 to 3 lines - preference for appending:
 showdiff(
   'a\n',
   'a\na\na\n'):
  'a\n'
  2 2 '' -> 'a\na\n'
 Diff 1 to 5 lines - preference for appending:
 showdiff(
   'a\n',
   'a\na\na\na\na\n'):
  'a\n'
  2 2 '' -> 'a\na\na\na\n'
 Diff 3 to 1 lines - preference for removing trailing lines:
 showdiff(
   'a\na\na\n',
   'a\n'):
  'a\n'
  2 6 'a\na\n' -> ''
 Diff 5 to 1 lines - preference for removing trailing lines:
 showdiff(
   'a\na\na\na\na\n',
   'a\n'):
  'a\n'
  2 10 'a\na\na\na\n' -> ''