upstream/mercurial-mirror Files · contrib/fuzz/dirs.cc

worker: Use buffered input from the pickle stream...

worker: Use buffered input from the pickle stream On Python 3, "pickle.load" will raise an exception ("_pickle.UnpicklingError: pickle data was truncated") when it gets a short read, i.e. it receives fewer bytes than it requested. On our build machine, Mercurial seems to frequently hit this problem while updating a mozilla-central clone iff it gets scheduled in batch mode. It is easy to trigger with: #wipe the workdir rm -rf * hg update null chrt -b 0 hg update default I've also written the following program, which demonstrates the core problem: from __future__ import print_function import io import os import pickle import time obj = {"a": 1, "b": 2} obj_data = pickle.dumps(obj) assert len(obj_data) > 10 rfd, wfd = os.pipe() pid = os.fork() if pid == 0: os.close(rfd) for _ in range(4): time.sleep(0.5) print("First write") os.write(wfd, obj_data[:10]) time.sleep(0.5) print("Second write") os.write(wfd, obj_data[10:]) os._exit(0) try: os.close(wfd) rfile = os.fdopen(rfd, "rb", 0) print("Reading") while True: try: obj_copy = pickle.load(rfile) assert obj == obj_copy except EOFError: break print("Success") finally: os.kill(pid, 15) The program reliably fails with Python 3.8 and succeeds with Python 2.7. Providing the unpickler with a buffered reader fixes the issue, so let "os.fdopen" create one. https://bugzilla.mozilla.org/show_bug.cgi?id=1604486 Differential Revision: https://phab.mercurial-scm.org/D8051

Augie Fackler - - Load All Authors

File last commit:

r44311:8766728d default


                r44718:cb52e619

stable

Download file

             dirs.cc
        
                    55 lines
            
             | 1.2 KiB
            
                | text/x-c
            
             |
                CppLexer
            
             / contrib / fuzz / dirs.cc
          
                    History
                
                 |
                  Source
                 | Raw
                 |Copy content
                 |Copy permalink

        Augie Fackler
    
fuzz: new fuzzer for dirs.c...

              r43420
            
      #include <Python.h>

      #include <assert.h>

      #include <stdlib.h>

      #include <unistd.h>

      #include "pyutil.h"

      #include <string>

      extern "C" {

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
      static PYCODETYPE *code;

        Augie Fackler
    
fuzz: new fuzzer for dirs.c...

              r43420
            
      extern "C" int LLVMFuzzerInitialize(int *argc, char ***argv)

      {

      	contrib::initpy(*argv[0]);

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
      	code = (PYCODETYPE *)Py_CompileString(R"py(

        Augie Fackler
    
fuzz: new fuzzer for dirs.c...

              r43420
            
      try:

        files = mdata.split('\n')

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
        d = parsers.dirs(files)

        Augie Fackler
    
fuzz: new fuzzer for dirs.c...

              r43420
            
        list(d)

        'a' in d

        if files:

          files[0] in d

      except Exception as e:

        pass

        # uncomment this print if you're editing this Python code

        # to debug failures.

        # print e

      )py",

        Augie Fackler
    
fuzz: add support for fuzzing under either Python 2 or 3...

              r44311
            
      	                                      "fuzzer", Py_file_input);

        Augie Fackler
    
fuzz: new fuzzer for dirs.c...

              r43420
            
      	return 0;

      }

      int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)

      {

      	// Don't allow fuzzer inputs larger than 100k, since we'll just bog

      	// down and not accomplish much.

      	if (Size > 100000) {

      		return 0;

      	}

      	PyObject *mtext =

      	    PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);

      	PyObject *locals = PyDict_New();

      	PyDict_SetItemString(locals, "mdata", mtext);

      	PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);

      	if (!res) {

      		PyErr_Print();

      	}

      	Py_XDECREF(res);

      	Py_DECREF(locals);

      	Py_DECREF(mtext);

      	return 0; // Non-zero return values are reserved for future use.

      }

      }

	Site-wide shortcuts
/	Use quick search box
g h	Goto home page
g g	Goto my private gists page
g G	Goto my public gists page
g 0-9	Goto bookmarked items from 0-9
n r	New repository page
n g	New gist page

	Repositories
g s	Goto summary page
g c	Goto changelog page
g f	Goto files page
g F	Goto files page with file search activated
g p	Goto pull requests page
g o	Goto repository settings
g O	Goto repository access permissions settings
t s	Toggle sidebar on some pages

Augie Fackler fuzz: new fuzzer for dirs.c...	r43420	#include <Python.h>
		#include <assert.h>
		#include <stdlib.h>
		#include <unistd.h>

		#include "pyutil.h"

		#include <string>

		extern "C" {

Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	static PYCODETYPE *code;
Augie Fackler fuzz: new fuzzer for dirs.c...	r43420
		extern "C" int LLVMFuzzerInitialize(int argc, char **argv)
		{
		contrib::initpy(*argv[0]);
Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	code = (PYCODETYPE *)Py_CompileString(R"py(
Augie Fackler fuzz: new fuzzer for dirs.c...	r43420	try:
		files = mdata.split('\n')
Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	d = parsers.dirs(files)
Augie Fackler fuzz: new fuzzer for dirs.c...	r43420	list(d)
		'a' in d
		if files:
		files[0] in d
		except Exception as e:
		pass
		# uncomment this print if you're editing this Python code
		# to debug failures.
		# print e
		)py",
Augie Fackler fuzz: add support for fuzzing under either Python 2 or 3...	r44311	"fuzzer", Py_file_input);
Augie Fackler fuzz: new fuzzer for dirs.c...	r43420	return 0;
		}

		int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
		{
		// Don't allow fuzzer inputs larger than 100k, since we'll just bog
		// down and not accomplish much.
		if (Size > 100000) {
		return 0;
		}
		PyObject *mtext =
		PyBytes_FromStringAndSize((const char *)Data, (Py_ssize_t)Size);
		PyObject *locals = PyDict_New();
		PyDict_SetItemString(locals, "mdata", mtext);
		PyObject *res = PyEval_EvalCode(code, contrib::pyglobals(), locals);
		if (!res) {
		PyErr_Print();
		}
		Py_XDECREF(res);
		Py_DECREF(locals);
		Py_DECREF(mtext);
		return 0; // Non-zero return values are reserved for future use.
		}
		}