From 6ecbcc7c198b5b61810dee2aeeba879e0e4b8042 Mon Sep 17 00:00:00 2001 From: Paul Dreik Date: Fri, 11 Sep 2020 23:46:22 +0200 Subject: [PATCH] add multi implementation fuzzer (#1162) This adds a fuzzer which parses the same input using all the available implementations (haswell, westmere, fallback on x64). This should get the otherwise uncovered sourcefiles (mostly fallback) to show up in the fuzz coverage. For instance, the fallback directory has only one line covered. As of the 20200909 report, 1866 lines are covered out of 4478. Also, it will detect if the implementations behave differently: by making sure they all succeed, or all error turning the parsed data into text again, should produce equal results While at it, I corrected some minor things: clean up building too many variants, run with forced implementation (closes #815 ) always store crashes as artefacts, good in case the fuzzer finds something return value of the fuzzer function should always be 0 reduce log spam introduce max size for the seed corpus and the CI fuzzer --- .github/workflows/fuzzers.yml | 72 ++++++++--------- fuzz/CMakeLists.txt | 7 +- fuzz/build_corpus.sh | 6 +- fuzz/build_fuzzer_variants.sh | 141 +++++----------------------------- fuzz/build_like_ossfuzz.sh | 2 +- fuzz/fuzz_dump.cpp | 2 +- fuzz/fuzz_dump_raw_tape.cpp | 2 +- fuzz/fuzz_implementations.cpp | 119 ++++++++++++++++++++++++++++ fuzz/fuzz_minify.cpp | 2 +- 9 files changed, 181 insertions(+), 172 deletions(-) create mode 100644 fuzz/fuzz_implementations.cpp diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index 3cc0a7015..12d293223 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -14,8 +14,10 @@ jobs: build: runs-on: ubuntu-latest env: - allfuzzers: parser dump dump_raw_tape print_json - artifactsprefix: -artifact_prefix=fuzzfailure/ + allfuzzers: parser dump dump_raw_tape print_json implementations + implementations: haswell westmere fallback + UBSAN_OPTIONS: halt_on_error=1 + MAXLEN: -max_len=4000 steps: - name: Install packages necessary for building run: | @@ -30,6 +32,8 @@ jobs: run: | fuzz/build_corpus.sh mv corpus.zip seed_corpus.zip + mkdir seedcorpus + unzip -q -d seedcorpus seed_corpus.zip - name: Download the corpus from the last run run: | wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar @@ -42,48 +46,28 @@ jobs: clang++ --version - name: Build all the variants run: fuzz/build_fuzzer_variants.sh - - name: Verify that the oss-fuzz seed corpus passes without problems - run: | - mkdir seedcorpus - unzip -q -d seedcorpus seed_corpus.zip - for buildvariant in noavx withavx; do - for fuzzer in $allfuzzers; do - build-ossfuzz-$buildvariant/fuzz/fuzz_$fuzzer seedcorpus -max_total_time=1 - done - done - - name: Run the fastest fuzzer to explore fast + - name: Run the fast fuzzer (release build, default implementation, to explore fast) run: | + set -eux for fuzzer in $allfuzzers; do mkdir -p out/$fuzzer # in case this is a new fuzzer, or corpus.tar is broken - build-ossfuzz-fast9/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=30 $artifactsprefix || touch failed - # make sure the failing output is visible in the log - if [ -e failed ] ; then - ls fuzzfailure/* |xargs -n1 base64 - exit 1 - fi + build-fast/fuzz/fuzz_$fuzzer out/$fuzzer seedcorpus -max_total_time=30 $MAXLEN done - - name: Run the other fuzzer variants for $fuzzer, with sanitizers etc + - name: Run the slow fuzzer (sanitizer+asserts, good at detecting errors) run: | - set -x + set -eux for fuzzer in $allfuzzers; do - build-ossfuzz-withavx/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=20 $artifactsprefix || touch failed - build-ossfuzz-noavx/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 $artifactsprefix || touch failed - build-ossfuzz-noavx9/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 $artifactsprefix || touch failed - if [ -e failed ] ; then - # make sure the failing output is visible in the log - ls fuzzfailure/* |xargs -n1 base64 - exit 1 - fi - echo disable msan runs, it fails inside the fuzzing engine and not the fuzzed code! - echo build-ossfuzz-msan-noavx9/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 -reload=0 $artifactsprefix - echo build-ossfuzz-msan-withavx9/fuzz/fuzz_$fuzzer out/$fuzzer -max_total_time=10 -reload=0 $artifactsprefix + for implementation in $implementations; do + export SIMDJSON_FORCE_IMPLEMENTATION=$implementation + build-sanitizers/fuzz/fuzz_$fuzzer out/$fuzzer seedcorpus -max_total_time=20 $MAXLEN + done echo now have $(ls out/$fuzzer |wc -l) files in corpus done - - name: Minimize the corpus with the fast fuzzer + - name: Minimize the corpus with the fast fuzzer on the default implementation run: | for fuzzer in $allfuzzers; do mkdir -p out/cmin/$fuzzer - build-ossfuzz-fast9/fuzz/fuzz_$fuzzer -merge=1 out/cmin/$fuzzer out/$fuzzer + build-fast/fuzz/fuzz_$fuzzer -merge=1 $MAXLEN out/cmin/$fuzzer out/$fuzzer seedcorpus rm -rf out/$fuzzer mv out/cmin/$fuzzer out/$fuzzer done @@ -97,16 +81,11 @@ jobs: with: name: corpus path: corpus.tar - - name: Run the corpus through valgrind (normal build) + - name: Run the minimized corpus through valgrind (replay build, default implementation) run: | for fuzzer in $allfuzzers; do - find out/$fuzzer -type f |sort|xargs valgrind build-plain-noavx/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer-noavx.txt - done - - name: Run the corpus through valgrind (noavx build) - run: | - for fuzzer in $allfuzzers; do - find out/$fuzzer -type f |sort|xargs valgrind build-plain-normal/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer-normal.txt - done + find out/$fuzzer -type f |sort|xargs -n40 valgrind build-replay/fuzz/fuzz_$fuzzer 2>&1|tee valgrind-$fuzzer.txt + done - name: Compress the valgrind output run: tar cf valgrind.tar valgrind-*.txt - name: Save valgrind output as a github artifact @@ -125,3 +104,14 @@ jobs: else echo "not on master, won't upload to bintray" fi + - name: Archive any crashes as an artifact + uses: actions/upload-artifact@v2 + if: always() + with: + name: crashes + path: | + crash-* + leak-* + timeout-* + if-no-files-found: ignore + diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index e158d553d..cf3b80843 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -54,10 +54,11 @@ if(ENABLE_FUZZING) set_property(TEST ${name} APPEND PROPERTY LABELS fuzz) endfunction() - implement_fuzzer(fuzz_parser) - implement_fuzzer(fuzz_minify) implement_fuzzer(fuzz_dump) - implement_fuzzer(fuzz_print_json) implement_fuzzer(fuzz_dump_raw_tape) + implement_fuzzer(fuzz_implementations) + implement_fuzzer(fuzz_minify) + implement_fuzzer(fuzz_parser) + implement_fuzzer(fuzz_print_json) endif() diff --git a/fuzz/build_corpus.sh b/fuzz/build_corpus.sh index c6e026783..6509741da 100755 --- a/fuzz/build_corpus.sh +++ b/fuzz/build_corpus.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Builds a corpus from all json files in the source directory. +# Builds a corpus from all small json files in the source directory. # The files are renamed to the sha1 of their content, and suffixed # .json. The files are zipped into a flat file named corpus.zip @@ -10,9 +10,9 @@ tmp=$(mktemp -d) root=$(readlink -f "$(dirname "$0")/..") -find $root -type f -name "*.json" | while read -r json; do +find $root -type f -size -4k -name "*.json" | while read -r json; do cp "$json" "$tmp"/$(sha1sum < "$json" |cut -f1 -d' ').json done -zip --junk-paths -r corpus.zip "$tmp" +zip --quiet --junk-paths -r corpus.zip "$tmp" rm -rf "$tmp" diff --git a/fuzz/build_fuzzer_variants.sh b/fuzz/build_fuzzer_variants.sh index eb5dd82c5..aff1e1301 100755 --- a/fuzz/build_fuzzer_variants.sh +++ b/fuzz/build_fuzzer_variants.sh @@ -12,163 +12,62 @@ unset CXX CC CFLAGS CXXFLAGS LDFLAGS me=$(basename $0) -# A reproduce build, without avx but otherwise as plain -# as it gets. No sanitizers or optimization. -variant=plain-noavx -if [ ! -d build-$variant ] ; then - mkdir build-$variant - cd build-$variant - - cmake .. \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ - -DSIMDJSON_COMPETITION=OFF \ - -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ - -DSIMDJSON_FUZZ_LINKMAIN=On \ - -DSIMDJSON_IMPLEMENTATION_HASWELL=0 - - ninja all_fuzzers - cd .. -fi +# common options +COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++-9 -DCMAKE_C_COMPILER=clang-9 -DSIMDJSON_BUILD_STATIC=On -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_GIT=Off" -# A reproduce build as plain as it gets. Everythings tunable is -# using the defaults. -variant=plain-normal +# A replay build, as plain as it gets. For use with valgrind/gdb. +variant=replay if [ ! -d build-$variant ] ; then mkdir build-$variant cd build-$variant cmake .. \ - -GNinja \ + $COMMON \ -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ - -DSIMDJSON_COMPETITION=OFF \ - -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ -DSIMDJSON_FUZZ_LINKMAIN=On ninja all_fuzzers cd .. fi -# a fuzzer with sanitizers, built with avx disabled. -variant=ossfuzz-noavx -if [ ! -d build-$variant ] ; then - - export CC=clang - export CXX="clang++" - export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx " - export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx" - export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" - - mkdir build-$variant - cd build-$variant - - cmake .. \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ - -DSIMDJSON_COMPETITION=OFF \ - -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ - -DSIMDJSON_FUZZ_LINKMAIN=Off \ - -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE \ - -DSIMDJSON_IMPLEMENTATION_HASWELL=0 - - ninja all_fuzzers - cd .. -fi +# A fuzzer with sanitizers. For improved capability to find bugs. +variant=sanitizers -# a fuzzer with sanitizers, built with avx disabled. -variant=ossfuzz-noavx9 -if which clang++-9 >/dev/null 2>&1 ; then if [ ! -d build-$variant ] ; then - export CC=clang-9 - export CXX="clang++-9" - export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx " - export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined -mno-avx2 -mno-avx" - export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" - mkdir build-$variant cd build-$variant cmake .. \ - -GNinja \ + $COMMON \ + -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \ + -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" \ -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ - -DSIMDJSON_COMPETITION=OFF \ - -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ -DSIMDJSON_FUZZ_LINKMAIN=Off \ - -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE \ - -DSIMDJSON_IMPLEMENTATION_HASWELL=0 + -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer" ninja all_fuzzers cd .. fi -else - echo "$me: WARNING clang++-9 not found, please install it to build $variant" -fi -# a fuzzer with sanitizers, default built -variant=ossfuzz-withavx -if [ ! -d build-$variant ] ; then - - export CC=clang - export CXX="clang++" - export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" - export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -fno-sanitize-recover=undefined" - export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" - - mkdir build-$variant - cd build-$variant - - cmake .. \ - -GNinja \ - -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ - -DSIMDJSON_COMPETITION=OFF \ - -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ - -DSIMDJSON_FUZZ_LINKMAIN=Off \ - -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE - - ninja all_fuzzers - cd .. -fi -# a fast fuzzer, for fast exploration -variant=ossfuzz-fast9 -if which clang++-9 >/dev/null 2>&1 ; then - if [ ! -d build-$variant ] ; then - export CC=clang-9 - export CXX="clang++-9" - export CFLAGS="-fsanitize=fuzzer-no-link -O3 -g" - export CXXFLAGS="-fsanitize=fuzzer-no-link -O3 -g" - export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" + +# A fast fuzzer, for fast exploration rather than finding bugs. +variant=fast + if [ ! -d build-$variant ] ; then mkdir build-$variant cd build-$variant cmake .. \ - -GNinja \ - -DCMAKE_BUILD_TYPE= \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ - -DSIMDJSON_COMPETITION=OFF \ - -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ + $COMMON \ + -DCMAKE_CXX_FLAGS="-fsanitize=fuzzer-no-link" \ + -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link" \ + -DCMAKE_BUILD_TYPE=Release \ -DSIMDJSON_FUZZ_LINKMAIN=Off \ - -DSIMDJSON_FUZZ_LDFLAGS=$LIB_FUZZING_ENGINE + -DSIMDJSON_FUZZ_LDFLAGS="-fsanitize=fuzzer" ninja all_fuzzers - cd .. fi -else - echo "$me: WARNING clang++-9 not found, please install it to build $variant" -fi - diff --git a/fuzz/build_like_ossfuzz.sh b/fuzz/build_like_ossfuzz.sh index 3b22b6ab6..9b69161d6 100755 --- a/fuzz/build_like_ossfuzz.sh +++ b/fuzz/build_like_ossfuzz.sh @@ -14,7 +14,7 @@ export OUT=$(pwd)/ossfuzz-out export CC=clang export CXX="clang++" export CFLAGS="-fsanitize=fuzzer-no-link" -export CXXFLAGS="-fsanitize=fuzzer-no-link" +export CXXFLAGS="-fsanitize=fuzzer-no-link,address,undefined -O3" export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" $ossfuzz diff --git a/fuzz/fuzz_dump.cpp b/fuzz/fuzz_dump.cpp index b70fb73f6..f520b1063 100644 --- a/fuzz/fuzz_dump.cpp +++ b/fuzz/fuzz_dump.cpp @@ -52,7 +52,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { simdjson::dom::element elem; auto error = parser.parse(Data, Size).get(elem); - if (error) { return 1; } + if (error) { return 0; } NulOStream os; //std::ostream& os(std::cout); print_json(os,elem); diff --git a/fuzz/fuzz_dump_raw_tape.cpp b/fuzz/fuzz_dump_raw_tape.cpp index 6b370f16d..43e1d0ad8 100644 --- a/fuzz/fuzz_dump_raw_tape.cpp +++ b/fuzz/fuzz_dump_raw_tape.cpp @@ -10,7 +10,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { simdjson::dom::parser parser; simdjson::dom::element elem; auto error = parser.parse(Data, Size).get(elem); - if (error) { return 1; } + if (error) { return 0; } NulOStream os; SIMDJSON_UNUSED auto dumpstatus = elem.dump_raw_tape(os); diff --git a/fuzz/fuzz_implementations.cpp b/fuzz/fuzz_implementations.cpp new file mode 100644 index 000000000..0a6b71b51 --- /dev/null +++ b/fuzz/fuzz_implementations.cpp @@ -0,0 +1,119 @@ +/* + * For fuzzing all of the implementations (haswell/fallback/westmere), + * finding any difference between the output of each which would + * indicate inconsistency. Also, it gets the non-default backend + * some fuzzing love. + * + * Copyright Paul Dreik 20200909 for the simdjson project. + */ + +#include "simdjson.h" +#include +#include +#include +#include +#include + + +// store each implementation along with it's intermediate results, +// which would make things easier to debug in case this fuzzer ever +// catches anything +struct Impl { + explicit Impl(const simdjson::implementation* im=nullptr) : impl(im),parser(),element(),error(),output(){} + //silence -Weffc++ + Impl(const Impl&)=delete; + Impl& operator=(const Impl&)=delete; + + const simdjson::implementation* impl; + simdjson::dom::parser parser; + simdjson::dom::element element; + simdjson::error_code error; + std::string output; +}; + +template +void showErrorAndAbort(Iterator first, Iterator last) { + auto it=first; + while(it!=last) { + std::cerr<<"Implementation: "<impl->name()<<"\tError:"<error<<'\n'; + it++; + } + std::cerr.flush(); + std::abort(); +} + +template +void showOutputAndAbort(Iterator first, Iterator last) { + + for(auto it=first;it!=last;++it) { + std::cerr<<"Implementation: "<impl->name()<<"\tOutput: "<output<<'\n'; + } + + // show the pairwise results + for(auto it1=first; it1!=last; ++it1) { + for(auto it2=it1; it2!=last; ++it2) { + if(it1!=it2) { + const bool matches=(it1->output==it2->output); + std::cerr<<"Implementation "<impl->name()<<" and "<impl->name()<<(matches?" match.":" do NOT match.")<<'\n'; + } + } + } + std::cerr.flush(); + std::abort(); +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { + + // make this dynamic, so it works regardless of how it was compiled + // or what hardware it runs on + constexpr std::size_t Nimplementations_max=3; + const std::size_t Nimplementations=simdjson::available_implementations.size(); + if(Nimplementations>Nimplementations_max) { + //there is another backend added, please bump Nimplementations_max! + std::abort(); + } + + // get pointers to the backend implementation + std::array implementations; + { + std::size_t i=0; + for(auto& e: simdjson::available_implementations) { + implementations[i++].impl=e; + } + } + + // let each implementation parse and store the result + std::size_t nerrors=0; + for(auto& e: implementations) { + simdjson::active_implementation=e.impl; + e.error=e.parser.parse(Data,Size).get(e.element); + if(e.error) { + ++nerrors; + } else { + std::ostringstream oss; + oss<