encoding: check overflow while calculating size of JSON escape buffer

The minimum input size to exploit is ~682MB (= INT_MAX / len('\\u0000') * 2)
on 32bit system, which isn't easy to achieve using Python str in 2GB process
address space, but probably doable.

mercurial/cext/charencode.c

@@ -294,11 +294,21 b' static Py_ssize_t jsonescapelen(const ch'
 				return -1;
 			}
 			esclen += jsonparanoidlentable[(unsigned char)c];
+			if (esclen < 0) {
+				PyErr_SetString(PyExc_MemoryError,
+						"overflow in jsonescapelen");
+				return -1;
+			}
 		}
 	} else {
 		for (i = 0; i < len; i++) {
 			char c = buf[i];
 			esclen += jsonlentable[(unsigned char)c];
+			if (esclen < 0) {
+				PyErr_SetString(PyExc_MemoryError,
+						"overflow in jsonescapelen");
+				return -1;
+			}
 		}
 	}
 
@@ -366,7 +376,7 b' PyObject *jsonescapeu8fast(PyObject *sel'
 	origlen = PyBytes_GET_SIZE(origstr);
 	esclen = jsonescapelen(origbuf, origlen, paranoid);
 	if (esclen < 0)
-		return NULL;  /* unsupported char found */
+		return NULL;  /* unsupported char found or overflow */
 	if (origlen == esclen) {
 		Py_INCREF(origstr);
 		return origstr;