diff --git a/.github/workflows/fuzzers.yml b/.github/workflows/fuzzers.yml index 60c369fb8..06421e199 100644 --- a/.github/workflows/fuzzers.yml +++ b/.github/workflows/fuzzers.yml @@ -44,6 +44,18 @@ jobs: path: dependencies/.cache key: ${{ hashFiles('dependencies/CMakeLists.txt') }} + - uses: actions/cache@v2 + id: cache-corpus + with: + path: out/ + key: corpus-${{ github.run_id }} + restore-keys: corpus- + + - name: show statistics for the cached corpus + run: | + echo number of files in github action corpus cache: + find out -type f |wc -l + - name: Create and prepare the initial seed corpus run: | fuzz/build_corpus.sh @@ -51,12 +63,6 @@ jobs: mkdir seedcorpus unzip -q -d seedcorpus seed_corpus.zip - - name: Download the corpus from the last run - run: | - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar - tar xf corpus.tar - rm corpus.tar - - name: List clang versions run: | ls /usr/bin/clang* @@ -70,7 +76,7 @@ jobs: run: | set -eux for fuzzer in $defaultimplfuzzers $implfuzzers; do - mkdir -p out/$fuzzer # in case this is a new fuzzer, or corpus.tar is broken + mkdir -p out/$fuzzer # in case this is a new fuzzer, or the github action cached corpus is broken # get input from everyone else (corpus cross pollination) others=$(find out -type d -not -name $fuzzer -not -name out -not -name cmin) build-fast/fuzz/fuzz_$fuzzer out/$fuzzer $others seedcorpus -max_total_time=30 $MAXLEN @@ -143,15 +149,6 @@ jobs: path: valgrind.tar if-no-files-found: ignore - - name: Upload the corpus and results to bintray if we are on master - if: ${{ github.event_name == 'schedule' }} - run: | - echo uploading each artifact twice, otherwise it will not be published - curl -T corpus.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/corpus.tar";publish=1;override=1" - curl -T corpus.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/corpus.tar";publish=1;override=1" - curl -T valgrind.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/valgrind.tar";publish=1;override=1" - curl -T valgrind.tar -upauldreik:${{ secrets.bintrayApiKey }} https://api.bintray.com/content/pauldreik/simdjson-fuzz-corpus/corpus/0/corpus/valgrind.tar";publish=1;override=1" - - name: Archive any crashes as an artifact uses: actions/upload-artifact@v2 if: always() diff --git a/fuzz/quick_check.sh b/fuzz/quick_check.sh index fce257274..83cf31dd2 100755 --- a/fuzz/quick_check.sh +++ b/fuzz/quick_check.sh @@ -23,8 +23,10 @@ done #download the corpus if it does not already exist if [ ! -d out ] ; then - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar - tar xf corpus.tar && rm corpus.tar + # ideally, we would download the github artifact but that requires being logged in which can not + # easily be fixed from this shell script. + echo "NOTE! please go to the artifacts page on https://github.com/simdjson/simdjson/actions/workflows/fuzzers.yml and download the latest corpus.tar.zip artifact manually to speed up fuzzing" + sleep 5s fi # By default, use the debug friendly variant since this script is intended