fuzz: add support for fuzzing under either Python 2 or 3

This was more of a hairball than I hoped, but it appears to work. The hg-py3
branch of my oss-fuzz fork on github has the remaining changes to switch us to
Python 3, but we may as well retain Python 2 fuzzing support for at least a
little while.

Differential Revision: https://phab.mercurial-scm.org/D7592

contrib/fuzz/dirs.cc

@@ -1,56 +1,55 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "pyutil.h"
 
 #include <string>
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import dirs
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 try:
   files = mdata.split('\n')
-  d = dirs(files)
+  d = parsers.dirs(files)
   list(d)
   'a' in d
   if files:
     files[0] in d
 except Exception as e:
   pass
   # uncomment this print if you're editing this Python code
   # to debug failures.
   # print e
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	// Don't allow fuzzer inputs larger than 100k, since we'll just bog
 	// down and not accomplish much.
 	if (Size > 100000) {
 		return 0;
 	}
 	PyObject *mtext =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "mdata", mtext);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(mtext);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/dirstate.cc

@@ -1,48 +1,47 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include <string>
 
 #include "pyutil.h"
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import parse_dirstate
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 try:
     dmap = {}
     copymap = {}
-    p = parse_dirstate(dmap, copymap, data)
+    p = parsers.parse_dirstate(dmap, copymap, data)
 except Exception as e:
     pass
     # uncomment this print if you're editing this Python code
     # to debug failures.
     # print e
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	PyObject *text =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "data", text);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(text);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/fm1readmarkers.cc

@@ -1,60 +1,59 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include <string>
 
 #include "pyutil.h"
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import fm1readmarkers
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 def maybeint(s, default):
     try:
         return int(s)
     except ValueError:
         return default
 try:
     parts = data.split('\0', 2)
     if len(parts) == 3:
         offset, stop, data = parts
     elif len(parts) == 2:
         stop, data = parts
         offset = 0
     else:
         offset = stop = 0
     offset, stop = maybeint(offset, 0), maybeint(stop, len(data))
-    fm1readmarkers(data, offset, stop)
+    parsers.fm1readmarkers(data, offset, stop)
 except Exception as e:
     pass
     # uncomment this print if you're editing this Python code
     # to debug failures.
     # print e
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	PyObject *text =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "data", text);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(text);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/fncache.cc

@@ -1,78 +1,69 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "pyutil.h"
 
 #include <iostream>
 #include <string>
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import (
-    isasciistr,
-    asciilower,
-    asciiupper,
-    encodedir,
-    pathencode,
-    lowerencode,
-)
-
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 try:
     for fn in (
-        isasciistr,
-        asciilower,
-        asciiupper,
-        encodedir,
-        pathencode,
-        lowerencode,
+        parsers.isasciistr,
+        parsers.asciilower,
+        parsers.asciiupper,
+        parsers.encodedir,
+        parsers.pathencode,
+        parsers.lowerencode,
     ):
         try:
             fn(data)
         except UnicodeDecodeError:
             pass  # some functions emit this exception
         except AttributeError:
             # pathencode needs hashlib, which fails to import because the time
             # module fails to import. We should try and fix that some day, but
             # for now we at least get coverage on non-hashencoded codepaths.
             if fn != pathencode:
                 raise
         # uncomment this for debugging exceptions
         # except Exception as e:
         #     raise Exception('%r: %r' % (fn, e))
 except Exception as e:
     pass
     # uncomment this print if you're editing this Python code
     # to debug failures.
     # print(e)
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	if (!code) {
 		std::cerr << "failed to compile Python code!" << std::endl;
 	}
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	PyObject *mtext =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "data", mtext);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(mtext);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/jsonescapeu8fast.cc

@@ -1,57 +1,55 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "pyutil.h"
 
 #include <iostream>
 #include <string>
 #include "FuzzedDataProvider.h"
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import jsonescapeu8fast
-
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 try:
-    jsonescapeu8fast(data, paranoid)
+    parsers.jsonescapeu8fast(data, paranoid)
 except Exception as e:
     pass
     # uncomment this print if you're editing this Python code
     # to debug failures.
     # print(e)
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	if (!code) {
 		std::cerr << "failed to compile Python code!" << std::endl;
 	}
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	FuzzedDataProvider provider(Data, Size);
 	bool paranoid = provider.ConsumeBool();
 	std::string remainder = provider.ConsumeRemainingBytesAsString();
 
 	PyObject *mtext = PyBytes_FromStringAndSize(
 	    (const char *)remainder.c_str(), remainder.size());
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "data", mtext);
 	PyDict_SetItemString(locals, "paranoid", paranoid ? Py_True : Py_False);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(mtext);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/manifest.cc

@@ -1,68 +1,67 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include "pyutil.h"
 
 #include <string>
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import lazymanifest
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 try:
-  lm = lazymanifest(mdata)
+  lm = parsers.lazymanifest(mdata)
   # iterate the whole thing, which causes the code to fully parse
   # every line in the manifest
   for e, _, _ in lm.iterentries():
       # also exercise __getitem__ et al
       lm[e]
       e in lm
       (e + 'nope') in lm
   lm[b'xyzzy'] = (b'\0' * 20, 'x')
   # do an insert, text should change
   assert lm.text() != mdata, "insert should change text and didn't: %r %r" % (lm.text(), mdata)
   cloned = lm.filtercopy(lambda x: x != 'xyzzy')
   assert cloned.text() == mdata, 'cloned text should equal mdata'
   cloned.diff(lm)
   del lm[b'xyzzy']
   cloned.diff(lm)
   # should be back to the same
   assert lm.text() == mdata, "delete should have restored text but didn't: %r %r" % (lm.text(), mdata)
 except Exception as e:
   pass
   # uncomment this print if you're editing this Python code
   # to debug failures.
   # print e
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	// Don't allow fuzzer inputs larger than 100k, since we'll just bog
 	// down and not accomplish much.
 	if (Size > 100000) {
 		return 0;
 	}
 	PyObject *mtext =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "mdata", mtext);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(mtext);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }

contrib/fuzz/pyutil.cc

@@ -1,54 +1,76 b''
 #include "pyutil.h"
 
 #include <iostream>
 #include <string>
 
 namespace contrib
 {
 
+#if PY_MAJOR_VERSION >= 3
+#define HG_FUZZER_PY3 1
+PyMODINIT_FUNC PyInit_parsers(void);
+#else
+PyMODINIT_FUNC initparsers(void);
+#endif
+
 static char cpypath[8192] = "\0";
 
 static PyObject *mainmod;
 static PyObject *globals;
 
-/* TODO: use Python 3 for this fuzzing? */
-PyMODINIT_FUNC initparsers(void);
-
 void initpy(const char *cselfpath)
 {
+#ifdef HG_FUZZER_PY3
+	const std::string subdir = "/sanpy/lib/python3.7";
+#else
 	const std::string subdir = "/sanpy/lib/python2.7";
+#endif
+
 	/* HACK ALERT: we need a full Python installation built without
 	   pymalloc and with ASAN, so we dump one in
 	   $OUT/sanpy/lib/python2.7. This helps us wire that up. */
 	std::string selfpath(cselfpath);
 	std::string pypath;
 	auto pos = selfpath.rfind("/");
 	if (pos == std::string::npos) {
 		char wd[8192];
 		if (!getcwd(wd, 8192)) {
 			std::cerr << "Failed to call getcwd: errno " << errno
 			          << std::endl;
 			exit(1);
 		}
 		pypath = std::string(wd) + subdir;
 	} else {
 		pypath = selfpath.substr(0, pos) + subdir;
 	}
 	strncpy(cpypath, pypath.c_str(), pypath.size());
 	setenv("PYTHONPATH", cpypath, 1);
 	setenv("PYTHONNOUSERSITE", "1", 1);
 	/* prevent Python from looking up users in the fuzz environment */
 	setenv("PYTHONUSERBASE", cpypath, 1);
+#ifdef HG_FUZZER_PY3
+	std::wstring wcpypath(pypath.begin(), pypath.end());
+	Py_SetPythonHome(wcpypath.c_str());
+#else
 	Py_SetPythonHome(cpypath);
+#endif
 	Py_InitializeEx(0);
 	mainmod = PyImport_AddModule("__main__");
 	globals = PyModule_GetDict(mainmod);
+
+#ifdef HG_FUZZER_PY3
+	PyObject *mod = PyInit_parsers();
+#else
 	initparsers();
+	PyObject *mod = PyImport_ImportModule("parsers");
+#endif
+
+	PyDict_SetItemString(globals, "parsers", mod);
 }
 
 PyObject *pyglobals()
 {
 	return globals;
 }
 
 } // namespace contrib

contrib/fuzz/pyutil.h

@@ -1,9 +1,15 b''
 #include <Python.h>
 
+#if PY_MAJOR_VERSION >= 3
+#define PYCODETYPE PyObject
+#else
+#define PYCODETYPE PyCodeObject
+#endif
+
 namespace contrib
 {
 
 void initpy(const char *cselfpath);
 PyObject *pyglobals();
 
 } /* namespace contrib */

contrib/fuzz/revlog.cc

@@ -1,62 +1,61 b''
 #include <Python.h>
 #include <assert.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 #include <string>
 
 #include "pyutil.h"
 
 extern "C" {
 
-static PyCodeObject *code;
+static PYCODETYPE *code;
 
 extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)
 {
 	contrib::initpy(*argv[0]);
-	code = (PyCodeObject *)Py_CompileString(R"py(
-from parsers import parse_index2
+	code = (PYCODETYPE *)Py_CompileString(R"py(
 for inline in (True, False):
     try:
-        index, cache = parse_index2(data, inline)
+        index, cache = parsers.parse_index2(data, inline)
         index.slicechunktodensity(list(range(len(index))), 0.5, 262144)
         index.stats()
         index.findsnapshots({}, 0)
         10 in index
         for rev in range(len(index)):
             index.reachableroots(0, [len(index)-1], [rev])
             node = index[rev][7]
             partial = index.shortest(node)
             index.partialmatch(node[:partial])
             index.deltachain(rev, None, True)
     except Exception as e:
         pass
         # uncomment this print if you're editing this Python code
         # to debug failures.
         # print e
 )py",
-	                                        "fuzzer", Py_file_input);
+	                                      "fuzzer", Py_file_input);
 	return 0;
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
 	// Don't allow fuzzer inputs larger than 60k, since we'll just bog
 	// down and not accomplish much.
 	if (Size > 60000) {
 		return 0;
 	}
 	PyObject *text =
 	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
 	PyObject *locals = PyDict_New();
 	PyDict_SetItemString(locals, "data", text);
 	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
 	if (!res) {
 		PyErr_Print();
 	}
 	Py_XDECREF(res);
 	Py_DECREF(locals);
 	Py_DECREF(text);
 	return 0; // Non-zero return values are reserved for future use.
 }
 }