upstream/mercurial-mirror Files · contrib/fuzz/jsonescapeu8fast.cc

worker: Use buffered input from the pickle stream...

worker: Use buffered input from the pickle stream On Python 3, "pickle.load" will raise an exception ("_pickle.UnpicklingError: pickle data was truncated") when it gets a short read, i.e. it receives fewer bytes than it requested. On our build machine, Mercurial seems to frequently hit this problem while updating a mozilla-central clone iff it gets scheduled in batch mode. It is easy to trigger with: #wipe the workdir rm -rf * hg update null chrt -b 0 hg update default I've also written the following program, which demonstrates the core problem: from __future__ import print_function import io import os import pickle import time obj = {"a": 1, "b": 2} obj_data = pickle.dumps(obj) assert len(obj_data) > 10 rfd, wfd = os.pipe() pid = os.fork() if pid == 0: os.close(rfd) for _ in range(4): time.sleep(0.5) print("First write") os.write(wfd, obj_data[:10]) time.sleep(0.5) print("Second write") os.write(wfd, obj_data[10:]) os._exit(0) try: os.close(wfd) rfile = os.fdopen(rfd, "rb", 0) print("Reading") while True: try: obj_copy = pickle.load(rfile) assert obj == obj_copy except EOFError: break print("Success") finally: os.kill(pid, 15) The program reliably fails with Python 3.8 and succeeds with Python 2.7. Providing the unpickler with a buffered reader fixes the issue, so let "os.fdopen" create one. https://bugzilla.mozilla.org/show_bug.cgi?id=1604486 Differential Revision: https://phab.mercurial-scm.org/D8051

Augie Fackler - - Load All Authors

File last commit:

r44311:8766728d default


                r44718:cb52e619

stable

Download file

             jsonescapeu8fast.cc
        
                    55 lines
            
             | 1.4 KiB
            
                | text/x-c
            
             |
                CppLexer
            
             / contrib / fuzz / jsonescapeu8fast.cc
          
                    History
                
                 |
                  Source
                 | Raw
                 |Copy content
                 |Copy permalink

        Augie Fackler
    
fuzz: new target to fuzz jsonescapeu8fast...

              r43423
            
      #include <Python.h>

      #include <assert.h>

      #include <stdlib.h>

      #include <unistd.h>

      #include "pyutil.h"

      #include <iostream>

      #include <string>

        Augie Fackler
    
fuzz: use a more standard approach to allow local builds of fuzzers...

              r44265
            
      #include "FuzzedDataProvider.h"

        Augie Fackler
    
fuzz: new target to fuzz jsonescapeu8fast...

              r43423
            
      extern "C" {

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
      static PYCODETYPE *code;

        Augie Fackler
    
fuzz: new target to fuzz jsonescapeu8fast...

              r43423
            
      extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)

      {

      	contrib::initpy(*argv[0]);

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
      	code = (PYCODETYPE *)Py_CompileString(R"py(

        Augie Fackler
    
fuzz: new target to fuzz jsonescapeu8fast...

              r43423
            
      try:

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
          parsers.jsonescapeu8fast(data, paranoid)

        Augie Fackler
    
fuzz: new target to fuzz jsonescapeu8fast...

              r43423
            
      except Exception as e:

          pass

          # uncomment this print if you're editing this Python code

          # to debug failures.

          # print(e)

      )py",

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
      	                                      "fuzzer", Py_file_input);

        Augie Fackler
    
fuzz: new target to fuzz jsonescapeu8fast...

              r43423
            
      	if (!code) {

      		std::cerr << "failed to compile Python code!" << std::endl;

      	}

      	return 0;

      }

      int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)

      {

      	FuzzedDataProvider provider(Data, Size);

      	bool paranoid = provider.ConsumeBool();

      	std::string remainder = provider.ConsumeRemainingBytesAsString();

      	PyObject *mtext = PyBytes_FromStringAndSize(

      	    (const char *)remainder.c_str(), remainder.size());

      	PyObject *locals = PyDict_New();

      	PyDict_SetItemString(locals, "data", mtext);

      	PyDict_SetItemString(locals, "paranoid", paranoid ? Py_True : Py_False);

      	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);

      	if (!res) {

      		PyErr_Print();

      	}

      	Py_XDECREF(res);

      	Py_DECREF(locals);

      	Py_DECREF(mtext);

      	return 0; // Non-zero return values are reserved for future use.

      }

      }

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

Augie Fackler fuzz: new target to fuzz jsonescapeu8fast...	r43423	#include <Python.h>
		#include <assert.h>
		#include <stdlib.h>
		#include <unistd.h>

		#include "pyutil.h"

		#include <iostream>
		#include <string>
Augie Fackler fuzz: use a more standard approach to allow local builds of fuzzers...	r44265	#include "FuzzedDataProvider.h"
Augie Fackler fuzz: new target to fuzz jsonescapeu8fast...	r43423
		extern "C" {

Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	static PYCODETYPE *code;
Augie Fackler fuzz: new target to fuzz jsonescapeu8fast...	r43423
		extern "C" int LLVMFuzzerInitialize(int argc, char **argv)
		{
		contrib::initpy(*argv[0]);
Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	code = (PYCODETYPE *)Py_CompileString(R"py(
Augie Fackler fuzz: new target to fuzz jsonescapeu8fast...	r43423	try:
Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	parsers.jsonescapeu8fast(data, paranoid)
Augie Fackler fuzz: new target to fuzz jsonescapeu8fast...	r43423	except Exception as e:
		pass
		# uncomment this print if you're editing this Python code
		# to debug failures.
		# print(e)
		)py",
Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	"fuzzer", Py_file_input);
Augie Fackler fuzz: new target to fuzz jsonescapeu8fast...	r43423	if (!code) {
		std::cerr << "failed to compile Python code!" << std::endl;
		}
		return 0;
		}

		int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
		{
		FuzzedDataProvider provider(Data, Size);
		bool paranoid = provider.ConsumeBool();
		std::string remainder = provider.ConsumeRemainingBytesAsString();

		PyObject *mtext = PyBytes_FromStringAndSize(
		(const char *)remainder.c_str(), remainder.size());
		PyObject *locals = PyDict_New();
		PyDict_SetItemString(locals, "data", mtext);
		PyDict_SetItemString(locals, "paranoid", paranoid ? Py_True : Py_False);
		PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
		if (!res) {
		PyErr_Print();
		}
		Py_XDECREF(res);
		Py_DECREF(locals);
		Py_DECREF(mtext);
		return 0; // Non-zero return values are reserved for future use.
		}
		}