diff --git a/.appveyor.yml b/.appveyor.yml index 7f016e85c..2d12fe3ae 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,19 +18,19 @@ environment: CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM? - job_name: VS2017 (Static, No Threads) image: Visual Studio 2017 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32) platform: Win32 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32, No Exceptions) platform: Win32 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2015 image: Visual Studio 2015 - CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly build_script: diff --git a/.circleci/config.yml b/.circleci/config.yml index 4feeed864..8af21d89c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -164,49 +164,49 @@ jobs: gcc10-perftest: description: Build and run performance tests on GCC 10 and AVX 2 with a cmake static build, this test performance regression executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_perftest ] gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake static build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ] clang6: description: Build and run tests on clang 6 and AVX 2 with a cmake static build executor: clang6 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test ] clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake static build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ] # libcpp libcpp-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake static build and libc++ executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON -DSIMDJSON_BUILD_STATIC=ON } + environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON -DBUILD_SHARED_LIBS=OFF } steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ] # sanitize sanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] threadsanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] threadsanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] nocheckeof-clang10: description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds @@ -220,12 +220,12 @@ jobs: dynamic-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build executor: gcc10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON } steps: [ cmake_test, cmake_install_test ] dynamic-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake dynamic build executor: clang10 - environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON } steps: [ cmake_test, cmake_install_test ] # unthreaded @@ -270,12 +270,12 @@ jobs: sanitize-haswell-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-haswell-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] workflows: diff --git a/.drone.yml b/.drone.yml index eafa52cbd..1412d3924 100644 --- a/.drone.yml +++ b/.drone.yml @@ -8,7 +8,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')" @@ -29,7 +29,7 @@ steps: CC: clang-6.0 CXX: clang++-6.0 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')" @@ -50,7 +50,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -77,7 +77,7 @@ steps: CC: clang-6.0 CXX: clang++-6.0 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - mkdir build @@ -100,7 +100,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -122,7 +122,7 @@ steps: environment: CC: clang-9 CXX: clang++-9 - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -142,7 +142,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -213,7 +213,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -237,7 +237,7 @@ steps: environment: CC: clang-6.0 CXX: clang++-6.0 - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -260,7 +260,7 @@ steps: CC: gcc CXX: g++ BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: - echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list @@ -281,7 +281,7 @@ steps: environment: CC: clang-6.0 CXX: clang++-6.0 - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON BUILD_FLAGS: -- -j CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly commands: @@ -302,7 +302,7 @@ steps: image: gcc:8 environment: BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CC: gcc CXX: g++ @@ -356,7 +356,7 @@ steps: CC: clang-9 CXX: clang++-9 BUILD_FLAGS: -- -j 4 - CMAKE_FLAGS: -GNinja -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -GNinja -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CXXFLAGS: -stdlib=libc++ commands: @@ -377,7 +377,7 @@ steps: CC: clang-9 CXX: clang++-9 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CXXFLAGS: -stdlib=libc++ commands: @@ -398,7 +398,7 @@ steps: CC: clang-7 CXX: clang++-7 BUILD_FLAGS: -- -j - CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON + CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly CXXFLAGS: -stdlib=libc++ commands: @@ -429,26 +429,28 @@ steps: - cmake $CMAKE_FLAGS .. - cmake --build . $BUILD_FLAGS - ctest $CTEST_FLAGS ---- -kind: pipeline -name: arm64-fuzz -platform: { os: linux, arch: arm64 } -steps: -- name: Build and run fuzzers shortly - image: ubuntu:20.04 - environment: - CC: clang - CXX: clang++ - DEBIAN_FRONTEND: noninteractive - ASAN_OPTIONS: detect_leaks=0 - commands: - - apt-get -qq update - - apt-get install -q -y clang cmake git wget zip ninja-build - - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar - - tar xf corpus.tar && rm corpus.tar - - fuzz/build_like_ossfuzz.sh - - mkdir -p common_out - - for fuzzer in build/fuzz/fuzz_* ; do echo $fuzzer;$fuzzer common_out out/* -max_total_time=40; done +############################## +# With bintray gone, it is clear how we should fuzz. +#--- +#kind: pipeline +#name: arm64-fuzz +#platform: { os: linux, arch: arm64 } +#steps: +#- name: Build and run fuzzers shortly +# image: ubuntu:20.04 +# environment: +# CC: clang +# CXX: clang++ +# DEBIAN_FRONTEND: noninteractive +# ASAN_OPTIONS: detect_leaks=0 +# commands: +# - apt-get -qq update +# - apt-get install -q -y clang cmake git wget zip ninja-build +# - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar +# - tar xf corpus.tar && rm corpus.tar +# - fuzz/build_like_ossfuzz.sh +# - mkdir -p common_out +# - for fuzzer in build/fuzz/fuzz_* ; do echo $fuzzer;$fuzzer common_out out/* -max_total_time=40; done --- kind: pipeline name: stylecheck diff --git a/.github/workflows/mingw-ci.yml b/.github/workflows/mingw-ci.yml index 74a9cb8b3..273c3880e 100644 --- a/.github/workflows/mingw-ci.yml +++ b/.github/workflows/mingw-ci.yml @@ -22,7 +22,7 @@ jobs: runs-on: windows-2016 env: - CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line + CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line CC: gcc CXX: g++ @@ -63,6 +63,6 @@ jobs: git --version mkdir build32 cd build32 - cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. cmake --build . --target acceptance_tests --verbose ctest -L acceptance --output-on-failure diff --git a/.github/workflows/mingw64-ci.yml b/.github/workflows/mingw64-ci.yml index b42813566..c032b8fa2 100644 --- a/.github/workflows/mingw64-ci.yml +++ b/.github/workflows/mingw64-ci.yml @@ -22,7 +22,7 @@ jobs: runs-on: windows-2016 env: - CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line + CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line CC: gcc CXX: g++ @@ -63,12 +63,12 @@ jobs: git --version mkdir build64 cd build64 - cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. cmake --build . --target acceptance_tests --verbose ctest -L acceptance --output-on-failure cd .. mkdir build64debug cd build64debug - cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. + cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF .. cmake --build . --target acceptance_tests --verbose ctest -L acceptance --output-on-failure diff --git a/.github/workflows/msys2.yml b/.github/workflows/msys2.yml index 734103dc3..d27d8bce9 100644 --- a/.github/workflows/msys2.yml +++ b/.github/workflows/msys2.yml @@ -52,6 +52,6 @@ jobs: run: | mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON .. + cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON .. cmake --build . --verbose ctest -j4 --output-on-failure -LE explicitonly diff --git a/.github/workflows/power-fuzz.yml b/.github/workflows/power-fuzz.yml index 356f2b517..77a988416 100644 --- a/.github/workflows/power-fuzz.yml +++ b/.github/workflows/power-fuzz.yml @@ -16,6 +16,16 @@ jobs: name: Build on ubuntu-20.04 ppc64le steps: - uses: actions/checkout@v2.1.0 + - uses: actions/cache@v2 + id: cache-corpus + with: + path: out/ + key: corpus-${{ github.run_id }} + restore-keys: corpus- + - name: show statistics for the cached corpus + run: | + echo number of files in github action corpus cache: + find out -type f |wc -l - uses: uraimo/run-on-arch-action@v2.0.5 name: Run commands id: runcmd @@ -42,8 +52,9 @@ jobs: cmake .. -GNinja \ -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX \ -DCMAKE_C_COMPILER=clang$CLANGSUFFIX \ - -DSIMDJSON_BUILD_STATIC=Off \ - -DENABLE_FUZZING=On \ + -DBUILD_SHARED_LIBS=OFF \ + -DSIMDJSON_DEVELOPER_MODE=ON \ + -DSIMDJSON_ENABLE_FUZZING=On \ -DSIMDJSON_COMPETITION=OFF \ -DSIMDJSON_GOOGLE_BENCHMARKS=OFF \ -DSIMDJSON_DISABLE_DEPRECATED_API=On \ @@ -55,8 +66,6 @@ jobs: cd .. builddir=build cmake --build $builddir - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar - tar xf corpus.tar && rm corpus.tar fuzzernames=$(cmake --build $builddir --target print_all_fuzzernames |tail -n1) for fuzzer in $fuzzernames ; do exe=$builddir/fuzz/$fuzzer diff --git a/.github/workflows/ubuntu18-checkperf.yml b/.github/workflows/ubuntu18-checkperf.yml index 29254230d..7836065aa 100644 --- a/.github/workflows/ubuntu18-checkperf.yml +++ b/.github/workflows/ubuntu18-checkperf.yml @@ -24,6 +24,6 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . --target checkperf && ctest --output-on-failure -R checkperf ubuntu18-checkperf.yml diff --git a/.github/workflows/ubuntu18.yml b/.github/workflows/ubuntu18.yml index 6436c26d3..bfbda67de 100644 --- a/.github/workflows/ubuntu18.yml +++ b/.github/workflows/ubuntu18.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && make install && diff --git a/.github/workflows/ubuntu20-checkperf.yml b/.github/workflows/ubuntu20-checkperf.yml index fcea860d5..bb17ac030 100644 --- a/.github/workflows/ubuntu20-checkperf.yml +++ b/.github/workflows/ubuntu20-checkperf.yml @@ -24,6 +24,6 @@ jobs: run: | mkdir build && cd build && - cmake -DCMAKE_CXX_FLAGS="-Werror=old-style-cast -pedantic -Wpedantic" -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DCMAKE_CXX_FLAGS="-Werror=old-style-cast -pedantic -Wpedantic" -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . --target checkperf && ctest --output-on-failure -R checkperf diff --git a/.github/workflows/ubuntu20-noexcept.yml b/.github/workflows/ubuntu20-noexcept.yml index 789e1c491..1eaafbfda 100644 --- a/.github/workflows/ubuntu20-noexcept.yml +++ b/.github/workflows/ubuntu20-noexcept.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && make install && diff --git a/.github/workflows/ubuntu20-nothread.yml b/.github/workflows/ubuntu20-nothread.yml index 2af570e64..60a9614b1 100644 --- a/.github/workflows/ubuntu20-nothread.yml +++ b/.github/workflows/ubuntu20-nothread.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && make install && diff --git a/.github/workflows/ubuntu20.yml b/.github/workflows/ubuntu20.yml index 1fcde2e60..53d613589 100644 --- a/.github/workflows/ubuntu20.yml +++ b/.github/workflows/ubuntu20.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build && - cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && + cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. && cmake --build . && ctest -j --output-on-failure -LE explicitonly && cmake --install . && diff --git a/.github/workflows/vs16-clang-ci.yml b/.github/workflows/vs16-clang-ci.yml index 3940caafa..6068d7aba 100644 --- a/.github/workflows/vs16-clang-ci.yml +++ b/.github/workflows/vs16-clang-ci.yml @@ -29,7 +29,7 @@ jobs: buildDirectory: "${{ github.workspace }}/../../_temp/windows" cmakeBuildType: Release buildWithCMake: true - cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=ON + cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=OFF buildWithCMakeArgs: --config Release - name: 'Run CTest' diff --git a/.github/workflows/vs16-ninja-ci.yml b/.github/workflows/vs16-ninja-ci.yml index 7b40bf2b7..2b36fdf48 100644 --- a/.github/workflows/vs16-ninja-ci.yml +++ b/.github/workflows/vs16-ninja-ci.yml @@ -29,7 +29,7 @@ jobs: buildDirectory: "${{ github.workspace }}/../../_temp/windows" cmakeBuildType: Release buildWithCMake: true - cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=ON + cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=OFF buildWithCMakeArgs: --config Release - name: 'Run CTest' run: ctest -C Release -LE explicitonly --output-on-failure diff --git a/.travis.yml b/.travis.yml index c436c5fc4..276d30b68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -176,7 +176,7 @@ install: export ASAN_OPTIONS="detect_leaks=0"; fi - if [[ "${STATIC}" == "on" ]]; then - export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_BUILD_STATIC=ON"; + export CMAKE_FLAGS="${CMAKE_FLAGS} -DBUILD_SHARED_LIBS=OFF"; fi - export CTEST_FLAGS="-j4 --output-on-failure -LE explicitonly" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fd62c0e2b..5dd0ebd22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ Pull requests are always invited. However, we ask that you follow these guidelin 3. Other types of changes must be clearly motivated. We openly discourage changes with no identifiable benefits. - Changes should be focused and minimal. You should change as few lines of code as possible. Please do not reformat or touch files needlessly. - New features must be accompanied of new tests, in general. -- Your code should pass our continuous-integration tests. It is your responsability to ensure that your proposal pass the tests. We do not merge pull requests that would break our build. +- Your code should pass our continuous-integration tests. It is your responsibility to ensure that your proposal pass the tests. We do not merge pull requests that would break our build. - An exception to this would be changes to non-code files, such as documentation and assets, or trivial changes to code, such as comments, where it is encouraged to explicitly ask for skipping a CI run using the `[skip ci]` prefix in your Pull Request title **and** in the first line of the most recent commit in a push. Example for such a commit: `[skip ci] Fixed typo in power_of_ten's docs` This benefits the project in such a way that the CI pipeline is not burdened by running jobs on changes that don't change any behavior in the code, which reduces wait times for other Pull Requests that do change behavior and require testing. diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 5e5679c97..c12399468 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -36,5 +36,6 @@ Nong Li Furkan Taşkale Brendan Knapp Danila Kutenin +Pavel Pavlov # if you have contributed to the project and your name does not # appear in this list, please let us know! diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 363512d86..000000000 --- a/Dockerfile +++ /dev/null @@ -1,88 +0,0 @@ -### -# -# Though simdjson requires only commonly available compilers and tools, it can -# be convenient to build it and test it inside a docker container: it makes it -# possible to test and benchmark simdjson under even relatively out-of-date -# Linux servers. It should also work under macOS and Windows, though not -# at native speeds, maybe. -# -# Assuming that you have a working docker server, this file -# allows you to build, test and benchmark simdjson. -# -# We build the library and associated files in the dockerbuild subdirectory. -# It may be necessary to delete it before creating the image: -# -# rm -r -f dockerbuild -# -# The need to delete the directory has nothing to do with docker per se: it is -# simply cleaner in CMake to start from a fresh directory. This is important: if you -# reuse the same directory with different configurations, you may get broken builds. -# -# -# Then you can build the image as follows: -# -# docker build -t simdjson --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) . -# -# Please note that the image does not contain a copy of the code. However, the image will contain the -# the compiler and the build system. This means that if you change the source code, after you have built -# the image, you won't need to rebuild the image. In fact, unless you want to try a different compiler, you -# do not need to ever rebuild the image, even if you do a lot of work on the source code. -# -# We specify the users to avoid having files owned by a privileged user (root) in our directory. Some -# people like to run their machine as the "root" user. We do not think it is cool. -# -# Then you need to build the project: -# -# docker run -v $(pwd):/project:Z simdjson -# -# Should you change a source file, you may need to call this command again. Because the output -# files are persistent between calls to this command (they reside in the dockerbuild directory), -# this command can be fast. -# -# Next you can test it as follows: -# -# docker run -it -v $(pwd):/project:Z simdjson sh -c "cd dockerbuild && ctest . --output-on-failure -LE explicitonly" -# -# The run the complete tests requires you to have built all of simdjson. -# -# Building all of simdjson takes a long time. Instead, you can build just one target: -# -# docker run -it -v $(pwd):/project:Z simdjson sh -c "[ -d dockerbuild ] || mkdir dockerbuild && cd dockerbuild && cmake .. && cmake --build . --target parse" -# -# Note that it is safe to remove dockerbuild before call the previous command, as the repository gets rebuild. It is also possible, by changing the command, to use a different directory name. -# -# You can run performance tests: -# -# docker run -it --privileged -v $(pwd):/project:Z simdjson sh -c "cd dockerbuild && for i in ../jsonexamples/*.json; do echo \$i; ./benchmark/parse \$i; done" -# -# The "--privileged" is recommended so you can get performance counters under Linux. -# -# You can also grab a fresh copy of simdjson and rebuild it, to make comparisons: -# -# docker run -it -v $(pwd):/project:Z simdjson sh -c "git clone https://github.com/simdjson/simdjson.git && cd simdjson && mkdir build && cd build && cmake .. && cmake --build . --target parse " -# -# Then you can run comparisons: -# -# docker run -it --privileged -v $(pwd):/project:Z simdjson sh -c "for i in jsonexamples/*.json; do echo \$i; dockerbuild/benchmark/parse \$i| grep GB| head -n 1; simdjson/build/benchmark/parse \$i | grep GB |head -n 1; done" -# -#### -FROM ubuntu:20.10 -################ -# We would prefer to use the conan io images but they do not support 64-bit ARM? The small gcc images appear to -# be broken on ARM. -# Furthermore, we would not expect users to frequently rebuild the container, so using ubuntu is probably fine. -############### -ARG USER_ID -ARG GROUP_ID -RUN apt-get update -qq -RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata -RUN apt-get install -y cmake g++ git -RUN mkdir project - -RUN addgroup --gid $GROUP_ID user; exit 0 -RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user; exit 0 -USER user -RUN gcc --version -WORKDIR /project - -CMD ["sh","-c","[ -d dockerbuild ] || mkdir dockerbuild && cd dockerbuild && cmake .. && cmake --build . "] diff --git a/HACKING.md b/HACKING.md index d78e999af..0fe713105 100644 --- a/HACKING.md +++ b/HACKING.md @@ -88,7 +88,7 @@ Other important files and directories: cd build cmake -D SIMDJSON_DEVELOPER_MODE=ON .. cmake --build . --config Release - benchmark/parse ../jsonexamples/twitter.json + benchmark/dom/parse ../jsonexamples/twitter.json ``` The last line becomes `./benchmark/Release/parse.exe ../jsonexample/twitter.json` under Windows. You may also use Google Benchmark: ```bash @@ -159,7 +159,7 @@ processor. At this point, we are require to use one of two main strategies. -1. On POSIX systems, the main compilers (LLVM clang, GNU gcc) allow us to use any intrinsic function after including the header, but they fail to inline the resulting instruction if the target processor does not support them. Because we compile for a generic processor, we would not be able to use most intrinsic functions. Thankfully, more recent versions of these compilers allow us to flag a region of code with a specific target, so that we can compile only some of the code with support for advanced instructions. Thus in our C++, one might notice macros like `TARGET_HASWELL`. It is then our responsability, at runtime, to only run the regions of code (that we call kernels) matching the properties of the runtime processor. The benefit of this approach is that the compiler not only let us use intrinsic functions, but it can also optimize the rest of the code in the kernel with advanced instructions we enabled. +1. On POSIX systems, the main compilers (LLVM clang, GNU gcc) allow us to use any intrinsic function after including the header, but they fail to inline the resulting instruction if the target processor does not support them. Because we compile for a generic processor, we would not be able to use most intrinsic functions. Thankfully, more recent versions of these compilers allow us to flag a region of code with a specific target, so that we can compile only some of the code with support for advanced instructions. Thus in our C++, one might notice macros like `TARGET_HASWELL`. It is then our responsibility, at runtime, to only run the regions of code (that we call kernels) matching the properties of the runtime processor. The benefit of this approach is that the compiler not only let us use intrinsic functions, but it can also optimize the rest of the code in the kernel with advanced instructions we enabled. 2. Under Visual Studio, the problem is somewhat simpler. Visual Studio will not only provide the intrinsic functions, but it will also allow us to use them. They will compile just fine. It is at runtime that they may cause a crash. So we do not need to mark regions of code for compilation toward advanced processors (e.g., with `TARGET_HASWELL` macros). The downside of the Visual Studio approach is that the compiler is not allowed to use advanced instructions others than those we specify. In principle, this means that Visual Studio has weaker optimization opportunities. diff --git a/README.md b/README.md index 05ab61919..ba5a30f67 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/simdjson.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&q=proj%3Asimdjson&can=2) + ![Ubuntu 18.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2018.04%20CI%20(GCC%207)/badge.svg) [![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html) ![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg) @@ -27,7 +27,6 @@ Table of Contents ----------------- * [Quick Start](#quick-start) - * [On Demand](#on-demand) * [Documentation](#documentation) * [Performance results](#performance-results) * [Real-world usage](#real-world-usage) @@ -186,4 +185,4 @@ For compilers that do not support [C++17](https://en.wikipedia.org/wiki/C%2B%2B1 For efficient number serialization, we bundle Florian Loitsch's implementation of the Grisu2 algorithm for binary to decimal floating-point numbers. The implementation was slightly modified by JSON for Modern C++ library. Both Florian Loitsch's implementation and JSON for Modern C++ are provided under the MIT license. -For runtime dispatching, we use some code from the PyTorch project licensed under 3-clause BSD. \ No newline at end of file +For runtime dispatching, we use some code from the PyTorch project licensed under 3-clause BSD. diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index ecbf41850..bb7635fec 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -1,20 +1,12 @@ +add_subdirectory(dom) + + include_directories( . linux ) link_libraries(simdjson-windows-headers test-data) link_libraries(simdjson) add_executable(benchfeatures benchfeatures.cpp) add_executable(get_corpus_benchmark get_corpus_benchmark.cpp) -add_executable(perfdiff perfdiff.cpp) -add_executable(parse parse.cpp) -add_executable(parse_stream parse_stream.cpp) -add_executable(statisticalmodel statisticalmodel.cpp) - -add_executable(parse_noutf8validation parse.cpp) -target_compile_definitions(parse_noutf8validation PRIVATE SIMDJSON_SKIPUTF8VALIDATION) -add_executable(parse_nonumberparsing parse.cpp) -target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARSING) -add_executable(parse_nostringparsing parse.cpp) -target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING) if (TARGET benchmark::benchmark) link_libraries(benchmark::benchmark) @@ -37,4 +29,3 @@ if (TARGET benchmark::benchmark) endif() endif() -include(checkperf.cmake) diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile deleted file mode 100644 index e26b3402d..000000000 --- a/benchmark/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# From the ROOT, run: -# docker build -t simdjsonbench -f benchmark/Dockerfile . && docker run --privileged -t simdjsonbench -FROM gcc:8.3 - -# # Build latest -# ENV latest_release=v0.2.1 -# WORKDIR /usr/src/$latest_release/ -# RUN git clone --depth 1 https://github.com/lemire/simdjson/ -b $latest_release . -# RUN make parse - -# # Build master -# WORKDIR /usr/src/master/ -# RUN git clone --depth 1 https://github.com/lemire/simdjson/ . -# RUN make parse - -# Build the current source -COPY . /usr/src/current/ -WORKDIR /usr/src/current/ -RUN make checkperf \ No newline at end of file diff --git a/benchmark/dom/CMakeLists.txt b/benchmark/dom/CMakeLists.txt new file mode 100644 index 000000000..0528cdf69 --- /dev/null +++ b/benchmark/dom/CMakeLists.txt @@ -0,0 +1,16 @@ +include_directories( .. ../linux ) +link_libraries(simdjson-windows-headers test-data) +link_libraries(simdjson) + +add_executable(perfdiff perfdiff.cpp) +add_executable(parse parse.cpp) +add_executable(parse_stream parse_stream.cpp) +add_executable(statisticalmodel statisticalmodel.cpp) + +add_executable(parse_noutf8validation parse.cpp) +target_compile_definitions(parse_noutf8validation PRIVATE SIMDJSON_SKIPUTF8VALIDATION) +add_executable(parse_nonumberparsing parse.cpp) +target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARSING) +add_executable(parse_nostringparsing parse.cpp) +target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING) +include(checkperf.cmake) diff --git a/benchmark/checkperf.cmake b/benchmark/dom/checkperf.cmake similarity index 99% rename from benchmark/checkperf.cmake rename to benchmark/dom/checkperf.cmake index 5c1cd6cf2..f599789bb 100644 --- a/benchmark/checkperf.cmake +++ b/benchmark/dom/checkperf.cmake @@ -76,7 +76,7 @@ if (Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER "2.1.4") AND (NOT CMAKE_G if (CMAKE_CONFIGURATION_TYPES) set(CHECKPERF_PARSE ${SIMDJSON_CHECKPERF_DIR}/build/benchmark/$/parse) else() - set(CHECKPERF_PARSE ${SIMDJSON_CHECKPERF_DIR}/build/benchmark/parse) + set(CHECKPERF_PARSE ${SIMDJSON_CHECKPERF_DIR}/build/benchmark/dom/parse) endif() add_custom_target( checkperf-parse ALL # TODO is ALL necessary? diff --git a/benchmark/parse.cpp b/benchmark/dom/parse.cpp similarity index 100% rename from benchmark/parse.cpp rename to benchmark/dom/parse.cpp diff --git a/benchmark/parse_stream.cpp b/benchmark/dom/parse_stream.cpp similarity index 100% rename from benchmark/parse_stream.cpp rename to benchmark/dom/parse_stream.cpp diff --git a/benchmark/perfdiff.cpp b/benchmark/dom/perfdiff.cpp similarity index 100% rename from benchmark/perfdiff.cpp rename to benchmark/dom/perfdiff.cpp diff --git a/benchmark/statisticalmodel.cpp b/benchmark/dom/statisticalmodel.cpp similarity index 100% rename from benchmark/statisticalmodel.cpp rename to benchmark/dom/statisticalmodel.cpp diff --git a/cmake/exception-flags.cmake b/cmake/exception-flags.cmake index 57600477c..243d1da57 100644 --- a/cmake/exception-flags.cmake +++ b/cmake/exception-flags.cmake @@ -19,7 +19,7 @@ editing CMAKE_CXX_FLAGS") # Here, we disable both with the - argument negation operator string(REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) - # Because we cannot change the flag above on an invidual target (yet), the + # Because we cannot change the flag above on an individual target (yet), the # definition below must similarly be added globally add_definitions(-D_HAS_EXCEPTIONS=0) elseif(CMAKE_COMPILER_IS_GNUCC) diff --git a/doc/basics.md b/doc/basics.md index 2f228591e..c016063df 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -459,7 +459,7 @@ select the value. If your keys contain the characters '/' or '~', they must be e '~0' respectively. An empty JSON Path refers to the whole document. We also extend the JSON Pointer support to include *relative* paths. -You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document. +You can apply a JSON path to any node and the path gets interpreted relatively, as if the current node were a whole JSON document. Consider the following example: @@ -816,7 +816,7 @@ Unlike `parser.parse`, both `parser.load_many(filename)` and `parser.parse_many( document at a time. 1. When calling `parser.load_many(filename)`, the file's content is loaded up in a memory buffer owned by the `parser`'s instance. Thus the file can be safely deleted after calling `parser.load_many(filename)` as the parser instance owns all of the data. -2. When calling `parser.parse_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.parse_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents. In constrast, calling `doc = parser.parse("[1,2,3]"_padded)` is safe because `parser.parse` eagerly parses the input. +2. When calling `parser.parse_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.parse_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents. In contrast, calling `doc = parser.parse("[1,2,3]"_padded)` is safe because `parser.parse` eagerly parses the input. Both `load_many` and `parse_many` take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents. diff --git a/doc/dom.md b/doc/dom.md index 35a5421ca..1bbfed9ba 100644 --- a/doc/dom.md +++ b/doc/dom.md @@ -213,7 +213,7 @@ select the value. If your keys contain the characters '/' or '~', they must be e '~0' respectively. An empty JSON Path refers to the whole document. We also extend the JSON Pointer support to include *relative* paths. -You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document. +You can apply a JSON path to any node and the path gets interpreted relatively, as if the current node were a whole JSON document. Consider the following example: @@ -632,4 +632,4 @@ simdjson::dom::parser parser; simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false); ```` -Setting the `realloc_if_needed` parameter `false` in this manner may lead to better performance since copies are avoided, but it requires that the user takes more responsibilities: the simdjson library cannot verify that the input buffer was padded with SIMDJSON_PADDING extra bytes. \ No newline at end of file +Setting the `realloc_if_needed` parameter `false` in this manner may lead to better performance since copies are avoided, but it requires that the user takes more responsibilities: the simdjson library cannot verify that the input buffer was padded with SIMDJSON_PADDING extra bytes. diff --git a/doc/ondemand_design.md b/doc/ondemand_design.md index b69e49712..92d414e15 100644 --- a/doc/ondemand_design.md +++ b/doc/ondemand_design.md @@ -636,7 +636,7 @@ direct raw ASCII comparisons: `key().raw()` provides direct access to the unesca You can compare `key()` with unescaped C strings (e.g., `key()=="test"`). It is expected that the provided string is a valid JSON string. Importantly, the C string must not contain an unescaped quote character (`"`). For speed, the comparison is done byte-by-byte -without handling the escaped caracters. +without handling the escaped characters. If you occasionally need to access and store the unescaped key values, you may use the `unescaped_key()` method. Once you have called `unescaped_key()` method, neither the `key()` nor the `unescaped_key()` methods should be called: the current field instance diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index e8dcc816e..e903e07a0 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -1,11 +1,11 @@ if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) - option(ENABLE_FUZZING "enable building the fuzzers" ON) + option(SIMDJSON_ENABLE_FUZZING "enable building the fuzzers" ON) else() - option(ENABLE_FUZZING "enable building the fuzzers" OFF) + option(SIMDJSON_ENABLE_FUZZING "enable building the fuzzers" OFF) endif() -if(ENABLE_FUZZING) +if(SIMDJSON_ENABLE_FUZZING) # First attempt at a fuzzer, using libFuzzer. # @@ -17,7 +17,7 @@ if(ENABLE_FUZZING) # export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined" # export CXX=clang++ # export CC=clang++ - # cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer + # cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_ENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer # ninja # settings this links in a main. useful for reproducing, @@ -25,7 +25,7 @@ if(ENABLE_FUZZING) # (note that libFuzzer can also reproduce, just pass it the files) # # Using this by default, means the fuzzers will be built as a part of the normal - # workflow, meaning they wont bitrot and will participate in refactoring etc. + # workflow, meaning they won't bitrot and will participate in refactoring etc. # option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On) diff --git a/fuzz/build_fuzzer_variants.sh b/fuzz/build_fuzzer_variants.sh index 6a532df14..7aa9e87e0 100755 --- a/fuzz/build_fuzzer_variants.sh +++ b/fuzz/build_fuzzer_variants.sh @@ -48,7 +48,7 @@ if [ -e $testfuzzer ] ; then rm $testfuzzer; fi # common options CXX_CLAGS_COMMON=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS" +COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS" # A replay build, as plain as it gets. For use with valgrind/gdb. variant=replay diff --git a/fuzz/measure_coverage.sh b/fuzz/measure_coverage.sh index 33dbfc944..480087ec3 100755 --- a/fuzz/measure_coverage.sh +++ b/fuzz/measure_coverage.sh @@ -23,8 +23,9 @@ if [ ! -d $bdir ] ; then cmake .. \ -GNinja \ -DCMAKE_BUILD_TYPE=Debug \ - -DSIMDJSON_BUILD_STATIC=On \ - -DENABLE_FUZZING=On \ + -DSIMDJSON_DEVELOPER_MODE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DSIMDJSON_ENABLE_FUZZING=On \ -DSIMDJSON_DISABLE_DEPRECATED_API=On \ -DSIMDJSON_FUZZ_LINKMAIN=On ninja all_fuzzers diff --git a/fuzz/ossfuzz.sh b/fuzz/ossfuzz.sh index b4cf9e7bd..18b680271 100755 --- a/fuzz/ossfuzz.sh +++ b/fuzz/ossfuzz.sh @@ -26,8 +26,9 @@ cd build cmake .. \ -GNinja \ -DCMAKE_BUILD_TYPE=Debug \ --DSIMDJSON_BUILD_STATIC=On \ --DENABLE_FUZZING=On \ +-DSIMDJSON_DEVELOPER_MODE=ON \ +-DBUILD_SHARED_LIBS=OFF \ +-DSIMDJSON_ENABLE_FUZZING=On \ -DSIMDJSON_COMPETITION=Off \ -DSIMDJSON_FUZZ_LINKMAIN=Off \ -DSIMDJSON_GOOGLE_BENCHMARKS=Off \ diff --git a/include/simdjson.h b/include/simdjson.h index 63e90c9a9..e65d8fdd8 100644 --- a/include/simdjson.h +++ b/include/simdjson.h @@ -4,7 +4,7 @@ /** * @mainpage * - * Check the [README.md](https://github.com/lemire/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second). + * Check the [README.md](https://github.com/simdjson/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second). * * Sample code. See https://github.com/simdjson/simdjson/blob/master/doc/basics.md for more examples. diff --git a/include/simdjson/arm64/bitmanipulation.h b/include/simdjson/arm64/bitmanipulation.h index b56e1a0ea..3502b3857 100644 --- a/include/simdjson/arm64/bitmanipulation.h +++ b/include/simdjson/arm64/bitmanipulation.h @@ -46,6 +46,40 @@ simdjson_really_inline int count_ones(uint64_t input_num) { return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); } + +#if defined(__GNUC__) // catches clang and gcc +/** + * ARM has a fast 64-bit "bit reversal function" that is handy. However, + * it is not generally available as an intrinsic function under Visual + * Studio (though this might be changing). Even under clang/gcc, we + * apparently need to invoke inline assembly. + */ +/* + * We use SIMDJSON_PREFER_REVERSE_BITS as a hint that algorithms that + * work well with bit reversal may use it. + */ +#define SIMDJSON_PREFER_REVERSE_BITS 1 + +/* reverse the bits */ +simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) { + uint64_t rev_bits; + __asm("rbit %0, %1" : "=r"(rev_bits) : "r"(input_num)); + return rev_bits; +} + +/** + * Flips bit at index 63 - lz. Thus if you have 'leading_zeroes' leading zeroes, + * then this will set to zero the leading bit. It is possible for leading_zeroes to be + * greating or equal to 63 in which case we trigger undefined behavior, but the output + * of such undefined behavior is never used. + **/ +NO_SANITIZE_UNDEFINED +simdjson_really_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) { + return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes); +} + +#endif + simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { #ifdef SIMDJSON_REGULAR_VISUAL_STUDIO *result = value1 + value2; diff --git a/include/simdjson/arm64/simd.h b/include/simdjson/arm64/simd.h index 81764cf9b..5b87c8ff8 100644 --- a/include/simdjson/arm64/simd.h +++ b/include/simdjson/arm64/simd.h @@ -364,7 +364,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x // Explicit conversion to/from unsigned // // Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same type. - // In theory, we could check this occurence with std::same_as and std::enabled_if but it is C++14 + // In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14 // and relatively ugly and hard to read. #ifndef SIMDJSON_REGULAR_VISUAL_STUDIO simdjson_really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {} diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index 752be7b06..ac5a81066 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -40,7 +40,7 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; * the input buf should be readable up to buf + SIMDJSON_PADDING * this is a stopgap; there should be a better description of the * main loop and its behavior that abstracts over this - * See https://github.com/lemire/simdjson/issues/174 + * See https://github.com/simdjson/simdjson/issues/174 */ constexpr size_t SIMDJSON_PADDING = 32; @@ -163,7 +163,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024; * the regular visual studio or clang under visual * studio, you still need to handle these issues. * - * Non-Windows sytems do not have this complexity. + * Non-Windows systems do not have this complexity. */ #if SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY // We set SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL under Windows. @@ -217,7 +217,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024; // now it is safe to trigger the include #include // though the file is there, it does not follow that we got the implementation #if defined(_LIBCPP_STRING_VIEW) -// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceeded C++17, +// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceded C++17, // included string_view. // This means that we have string_view *even though* we may not have C++17. #define SIMDJSON_HAS_STRING_VIEW diff --git a/include/simdjson/dom/document-inl.h b/include/simdjson/dom/document-inl.h index 31b43a7ef..b5db40a04 100644 --- a/include/simdjson/dom/document-inl.h +++ b/include/simdjson/dom/document-inl.h @@ -37,7 +37,7 @@ inline error_code document::allocate(size_t capacity) noexcept { // need a capacity of at least capacity + 1, but it is also possible to do // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" //where capacity + 1 tape elements are - // generated, see issue https://github.com/lemire/simdjson/issues/345 + // generated, see issue https://github.com/simdjson/simdjson/issues/345 size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64); // a document with only zero-length strings... could have capacity/3 string // and we would need capacity/3 * 5 bytes on the string buffer diff --git a/include/simdjson/dom/document_stream.h b/include/simdjson/dom/document_stream.h index 2e30437b5..1cce78379 100644 --- a/include/simdjson/dom/document_stream.h +++ b/include/simdjson/dom/document_stream.h @@ -123,7 +123,7 @@ public: using iterator_category = std::input_iterator_tag; /** - * Default contructor. + * Default constructor. */ simdjson_really_inline iterator() noexcept; /** diff --git a/include/simdjson/dom/parsedjson_iterator.h b/include/simdjson/dom/parsedjson_iterator.h index 387354958..b5846048e 100644 --- a/include/simdjson/dom/parsedjson_iterator.h +++ b/include/simdjson/dom/parsedjson_iterator.h @@ -208,7 +208,7 @@ public: // throughout return true if we can do the navigation, false // otherwise - // Withing a given scope (series of nodes at the same depth within either an + // Within a given scope (series of nodes at the same depth within either an // array or an object), we move forward. // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { // and [. At the object ({) or at the array ([), you can issue a "down" to diff --git a/include/simdjson/dom/parser.h b/include/simdjson/dom/parser.h index b5747d250..998fc584b 100644 --- a/include/simdjson/dom/parser.h +++ b/include/simdjson/dom/parser.h @@ -536,7 +536,7 @@ public: /** * @private return an error code corresponding to the last parsing attempt, see - * simdjson.h will return UNITIALIZED if no parsing was attempted + * simdjson.h will return UNINITIALIZED if no parsing was attempted */ [[deprecated("Use the result of parser.parse() instead")]] inline int get_error_code() const noexcept; diff --git a/include/simdjson/dom/serialization.h b/include/simdjson/dom/serialization.h index 29227f956..84eaf7d26 100644 --- a/include/simdjson/dom/serialization.h +++ b/include/simdjson/dom/serialization.h @@ -37,7 +37,7 @@ public: inline void append(simdjson::dom::element value); /** Append an array to the builder (to be printed) **/ inline void append(simdjson::dom::array value); - /** Append an objet to the builder (to be printed) **/ + /** Append an object to the builder (to be printed) **/ inline void append(simdjson::dom::object value); /** Reset the builder (so that it would print the empty string) **/ simdjson_really_inline void clear(); @@ -149,7 +149,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result 2046)) { - // We have an infinte value!!! We could actually throw an error here if we could. + // We have an infinite value!!! We could actually throw an error here if we could. return false; } d = to_double(mantissa, real_exponent, negative); @@ -454,7 +454,7 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg // we could extend our code by using a 128-bit integer instead // of a 64-bit integer. However, this is uncommon in practice. // - // 9999999999999999999 < 2**64 so we can accomodate 19 digits. + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. // If we have a decimal separator, then digit_count - 1 is the number of digits, but we // may not have a decimal separator! if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { diff --git a/include/simdjson/generic/ondemand/raw_json_string.h b/include/simdjson/generic/ondemand/raw_json_string.h index edb3b1776..18b55568a 100644 --- a/include/simdjson/generic/ondemand/raw_json_string.h +++ b/include/simdjson/generic/ondemand/raw_json_string.h @@ -75,7 +75,7 @@ public: * long strings. * * If target is a compile-time constant, and your compiler likes you, - * you should be able to do the following without performance penatly... + * you should be able to do the following without performance penalty... * * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); * s.unsafe_is_equal(target); @@ -89,7 +89,7 @@ public: * the caller is responsible for this check. See is_free_from_unescaped_quote. * * If target is a compile-time constant, and your compiler likes you, - * you should be able to do the following without performance penatly... + * you should be able to do the following without performance penalty... * * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); * s.unsafe_is_equal(target); diff --git a/include/simdjson/generic/ondemand/serialization.h b/include/simdjson/generic/ondemand/serialization.h index b2b14e77d..df968e034 100644 --- a/include/simdjson/generic/ondemand/serialization.h +++ b/include/simdjson/generic/ondemand/serialization.h @@ -16,7 +16,7 @@ public: inline simdjson::error_code append(value element) noexcept; /** Append an array to the builder (to be printed) **/ inline simdjson::error_code append(array value) noexcept; - /** Append an objet to the builder (to be printed) **/ + /** Append an object to the builder (to be printed) **/ inline simdjson::error_code append(object value) noexcept; /** Append a field to the builder (to be printed) **/ inline simdjson::error_code append(field value) noexcept; @@ -134,7 +134,7 @@ inline std::ostream& operator<<(std::ostream& out, document& value) { * Print JSON to an output stream. * * @param out The output stream. - * @param value The objet. + * @param value The object. * @throw if there is an error with the underlying output stream. simdjson itself will not throw. */ #if SIMDJSON_EXCEPTIONS diff --git a/jsonexamples/small/jsoniter_scala/README.md b/jsonexamples/small/jsoniter_scala/README.md index 7ca1f8338..108850dc2 100644 --- a/jsonexamples/small/jsoniter_scala/README.md +++ b/jsonexamples/small/jsoniter_scala/README.md @@ -1,7 +1,7 @@ Files from https://github.com/plokhotnyuk/jsoniter-scala/tree/master/jsoniter-scala-benchmark/src/main/resources/com/github/plokhotnyuk/jsoniter_scala/benchmark See issue: -https://github.com/lemire/simdjson/issues/70 +https://github.com/simdjson/simdjson/issues/70 The files che-*.geo.json are number-parsing stress tests. diff --git a/src/generic/stage1/json_structural_indexer.h b/src/generic/stage1/json_structural_indexer.h index 781781015..915b33e74 100644 --- a/src/generic/stage1/json_structural_indexer.h +++ b/src/generic/stage1/json_structural_indexer.h @@ -31,8 +31,62 @@ public: // it helps tremendously. if (bits == 0) return; - int cnt = static_cast(count_ones(bits)); +#if defined(SIMDJSON_PREFER_REVERSE_BITS) + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + uint64_t rev_bits = reverse_bits(bits); + int cnt = static_cast(count_ones(bits)); + int i = 0; + // Do the first 8 all together + for (; i<8; i++) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } + // Do the next 8 all together (we hope in most cases it won't happen at all + // and the branch is easily predicted). + if (simdjson_unlikely(cnt > 8)) { + i = 8; + for (; i<16; i++) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } + + + // Most files don't have 16+ structurals per block, so we take several basically guaranteed + // branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :) + // or the start of a value ("abc" true 123) every four characters. + if (simdjson_unlikely(cnt > 16)) { + i = 16; + while (rev_bits != 0) { + int lz = leading_zeroes(rev_bits); + this->tail[i++] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } + } + } + this->tail += cnt; +#else // SIMDJSON_PREFER_REVERSE_BITS + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + int cnt = static_cast(count_ones(bits)); // Do the first 8 all together for (int i=0; i<8; i++) { this->tail[i] = idx + trailing_zeroes(bits); @@ -61,6 +115,7 @@ public: } this->tail += cnt; +#endif } }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2663f3700..fe0f9dbdf 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,189 +1,27 @@ # Most tests need test data, and many need windows headers. link_libraries(simdjson-internal-flags test-data simdjson-windows-headers) include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake) - +add_subdirectory(dom) add_subdirectory(ondemand) -# -# These tests explicitly do #include "simdjson.cpp" so they can override stuff -# -if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) - add_cpp_test(numberparsingcheck LABELS acceptance per_implementation) - simdjson_apply_props(numberparsingcheck) - target_link_libraries(numberparsingcheck PRIVATE simdjson-windows-headers) - target_compile_definitions(numberparsingcheck PRIVATE NOMINMAX) - target_include_directories( - numberparsingcheck - PRIVATE "${PROJECT_SOURCE_DIR}/src" - ) - add_cpp_test(stringparsingcheck LABELS acceptance per_implementation) - simdjson_apply_props(numberparsingcheck) - target_link_libraries(stringparsingcheck PRIVATE simdjson-windows-headers) - target_compile_definitions(stringparsingcheck PRIVATE NOMINMAX) - target_include_directories( - stringparsingcheck - PRIVATE "${PROJECT_SOURCE_DIR}/src" - ) -endif() # All remaining tests link with simdjson proper link_libraries(simdjson) -if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) - add_cpp_test(random_string_number_tests LABELS dom acceptance per_implementation) -endif() -add_cpp_test(basictests LABELS dom acceptance per_implementation) -add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation) -add_cpp_test(document_tests LABELS dom acceptance per_implementation) -add_cpp_test(errortests LABELS dom acceptance per_implementation) -add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation) -add_cpp_test(integer_tests LABELS dom acceptance per_implementation) -add_cpp_test(jsoncheck LABELS dom acceptance per_implementation) -add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation) -add_cpp_test(parse_many_test LABELS dom acceptance per_implementation) -add_cpp_test(pointercheck LABELS dom acceptance per_implementation) # https://tools.ietf.org/html/rfc6901 -add_cpp_test(unicode_tests LABELS dom acceptance per_implementation) -add_cpp_test(trivially_copyable_test LABELS dom acceptance per_implementation) +add_cpp_test(unicode_tests LABELS dom acceptance per_implementation) add_cpp_test(minify_tests LABELS other acceptance per_implementation) add_cpp_test(padded_string_tests LABELS other acceptance ) - -find_program(BASH bash) - -# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to -# map app paths to their "mounted" equivalent (e.g., /mnt/c/...). So even if you have bash under Windows, extra work would be -# required to make things work robustly. Simply put: bash is not quite portable. - -# Script tests -if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts are not robust enough to run under Windows even if bash is available - # - # json2json test - # - add_test( - NAME testjson2json - COMMAND ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh - WORKING_DIRECTORY $ - ) - set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json) - set_property(TEST testjson2json APPEND PROPERTY LABELS per_implementation) - add_dependencies(per_implementation_tests minify) - - # - # Competition parse test - # - if ((SIMDJSON_COMPETITION) AND (!SIMDJSON_SANITIZE)) - # It looks like RapidJSON does not pass the sanitizer under some conditions (Clang 10) - add_executable(allparserscheckfile allparserscheckfile.cpp) - add_dependencies(competition_tests allparserscheckfile) - add_dependencies(per_implementation_tests allparserscheckfile) - target_link_libraries(allparserscheckfile PRIVATE competition-all) - - add_test(issue150 ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh) - set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation competition) - add_dependencies(all_tests allparserscheckfile) - endif() - - # - # json2json tool test: check that json2json can parse twitter.json - # - - # This tests validates that the implementation is what we think it is if we get passed - # SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing - add_cpp_test(checkimplementation LABELS other per_implementation) - - add_test(NAME json2json COMMAND $ ${EXAMPLE_JSON}) - set_property(TEST json2json APPEND PROPERTY LABELS acceptance per_implementation) - add_dependencies(acceptance_tests json2json) - add_dependencies(per_implementation_tests json2json) - - # - # SIMDJSON_FORCE_IMPLEMENTATION tests: run json2json with SIMDJSON - # - if (SIMDJSON_IMPLEMENTATION MATCHES "fallback") - add_test( - NAME simdjson_force_implementation - COMMAND - ${CMAKE_COMMAND} -E env - SIMDJSON_FORCE_IMPLEMENTATION=fallback - $ - ) - endif() - add_test( - NAME simdjson_force_implementation_error - COMMAND - ${CMAKE_COMMAND} -E env - SIMDJSON_FORCE_IMPLEMENTATION=doesnotexist - $ ${EXAMPLE_JSON} - ) - set_tests_properties(simdjson_force_implementation_error PROPERTIES WILL_FAIL TRUE) -endif() - -# -# Compile-only tests with simdjson flags on -# - -# Don't add the tests if we're on VS2017 or older; they don't succeed. -if(NOT (MSVC AND MSVC_VERSION LESS 1920)) - if(SIMDJSON_EXCEPTIONS) - add_cpp_test(readme_examples COMPILE_ONLY LABELS acceptance) - add_cpp_test(readme_examples11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp) - set_target_properties(readme_examples11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF) - endif() - - add_cpp_test(readme_examples_noexceptions COMPILE_ONLY LABELS acceptance) - - add_cpp_test(readme_examples_noexceptions11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples_noexceptions.cpp) - set_target_properties(readme_examples_noexceptions11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF) - - # Compile tests that *should fail* - add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp) - target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0) -endif() - - -if(MSVC) - add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event - COMMAND ${CMAKE_COMMAND} -E echo "$" - COMMAND ${CMAKE_COMMAND} -E echo "$" +if(MSVC AND BUILD_SHARED_LIBS) + # Copy the simdjson dll into the tests directory + add_custom_command(TARGET unicode_tests POST_BUILD # Adds a post-build event COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..." "$" # <--this is in-file - "$") # <--this is out-file path + "$") # <--this is out-file path endif() -# Copy the simdjson dll into the tests directory -if(MSVC) - add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event - COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..." - "$" # <--this is in-file - "$") # <--this is out-file path -endif() - -## Next bit should not be needed! -#if(CMAKE_INTERPROCEDURAL_OPTIMIZATION) -# next line is a workaround for an odr-violation in basictests regarding the globals 0x432a40 and 0x52045c under clang -#set_tests_properties(basictests PROPERTIES -# ENVIRONMENT ASAN_OPTIONS="detect_odr_violation=0") -#endif() - -## This causes problems -# add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp) -# target_link_libraries(singleheader simdjson simdjson-flags) -# add_test(singleheader singleheader) - -if(NOT MSVC) - ###### - # This tests is to guard us against ever again removing the soversion - # number from the library. See Bug - # https://github.com/simdjson/simdjson/issues/1014 - ##### - get_target_property(REPORTED_SIMDJSON_SOVERSION simdjson SOVERSION) - if(NOT ${REPORTED_SIMDJSON_SOVERSION} STREQUAL ${SIMDJSON_LIB_SOVERSION}) - message(FATAL_ERROR "The library target does not have the proper soversion information." ) - endif() - get_target_property(REPORTED_SIMDJSON_VERSION simdjson VERSION) - if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION}) - message(FATAL_ERROR "The library target does not have the proper version information." ) - endif() -endif() +# This tests validates that the implementation is what we think it is if we get passed +# SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing +add_cpp_test(checkimplementation LABELS other per_implementation) add_subdirectory(compilation_failure_tests) diff --git a/tests/dom/CMakeLists.txt b/tests/dom/CMakeLists.txt new file mode 100644 index 000000000..e978e25d6 --- /dev/null +++ b/tests/dom/CMakeLists.txt @@ -0,0 +1,155 @@ + +# +# These tests explicitly do #include "simdjson.cpp" so they can override stuff +# +if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) + add_cpp_test(numberparsingcheck LABELS acceptance per_implementation) + simdjson_apply_props(numberparsingcheck) + target_link_libraries(numberparsingcheck PRIVATE simdjson-windows-headers) + target_compile_definitions(numberparsingcheck PRIVATE NOMINMAX) + target_include_directories( + numberparsingcheck + PRIVATE "${PROJECT_SOURCE_DIR}/src" + ) + add_cpp_test(stringparsingcheck LABELS acceptance per_implementation) + simdjson_apply_props(numberparsingcheck) + target_link_libraries(stringparsingcheck PRIVATE simdjson-windows-headers) + target_compile_definitions(stringparsingcheck PRIVATE NOMINMAX) + target_include_directories( + stringparsingcheck + PRIVATE "${PROJECT_SOURCE_DIR}/src" + ) +endif() + +link_libraries(simdjson) +include_directories(..) + +if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) + add_cpp_test(random_string_number_tests LABELS dom acceptance per_implementation) +endif() +add_cpp_test(basictests LABELS dom acceptance per_implementation) +add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation) +add_cpp_test(document_tests LABELS dom acceptance per_implementation) +add_cpp_test(errortests LABELS dom acceptance per_implementation) +add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation) +add_cpp_test(integer_tests LABELS dom acceptance per_implementation) +add_cpp_test(jsoncheck LABELS dom acceptance per_implementation) +add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation) +add_cpp_test(parse_many_test LABELS dom acceptance per_implementation) +add_cpp_test(pointercheck LABELS dom acceptance per_implementation) # https://tools.ietf.org/html/rfc6901 +add_cpp_test(trivially_copyable_test LABELS dom acceptance per_implementation) + + + +find_program(BASH bash) + +# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to +# map app paths to their "mounted" equivalent (e.g., /mnt/c/...). So even if you have bash under Windows, extra work would be +# required to make things work robustly. Simply put: bash is not quite portable. + +# Script tests +if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts are not robust enough to run under Windows even if bash is available + # + # json2json test + # + add_test( + NAME testjson2json + COMMAND ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh + WORKING_DIRECTORY $ + ) + set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json) + set_property(TEST testjson2json APPEND PROPERTY LABELS per_implementation) + add_dependencies(per_implementation_tests minify) + + # + # Competition parse test + # + if ((SIMDJSON_COMPETITION) AND (!SIMDJSON_SANITIZE)) + # It looks like RapidJSON does not pass the sanitizer under some conditions (Clang 10) + add_executable(allparserscheckfile allparserscheckfile.cpp) + add_dependencies(competition_tests allparserscheckfile) + add_dependencies(per_implementation_tests allparserscheckfile) + target_link_libraries(allparserscheckfile PRIVATE competition-all) + + add_test(issue150 ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh) + set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation competition) + add_dependencies(all_tests allparserscheckfile) + endif() + + # + # json2json tool test: check that json2json can parse twitter.json + # + + + add_test(NAME json2json COMMAND $ ${EXAMPLE_JSON}) + set_property(TEST json2json APPEND PROPERTY LABELS acceptance per_implementation) + add_dependencies(acceptance_tests json2json) + add_dependencies(per_implementation_tests json2json) + + # + # SIMDJSON_FORCE_IMPLEMENTATION tests: run json2json with SIMDJSON + # + if (SIMDJSON_IMPLEMENTATION MATCHES "fallback") + add_test( + NAME simdjson_force_implementation + COMMAND + ${CMAKE_COMMAND} -E env + SIMDJSON_FORCE_IMPLEMENTATION=fallback + $ + ) + endif() + add_test( + NAME simdjson_force_implementation_error + COMMAND + ${CMAKE_COMMAND} -E env + SIMDJSON_FORCE_IMPLEMENTATION=doesnotexist + $ ${EXAMPLE_JSON} + ) + set_tests_properties(simdjson_force_implementation_error PROPERTIES WILL_FAIL TRUE) +endif() + +# +# Compile-only tests with simdjson flags on +# + +# Don't add the tests if we're on VS2017 or older; they don't succeed. +if(NOT (MSVC AND MSVC_VERSION LESS 1920)) + if(SIMDJSON_EXCEPTIONS) + add_cpp_test(readme_examples COMPILE_ONLY LABELS acceptance) + add_cpp_test(readme_examples11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp) + set_target_properties(readme_examples11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF) + endif() + + add_cpp_test(readme_examples_noexceptions COMPILE_ONLY LABELS acceptance) + + add_cpp_test(readme_examples_noexceptions11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples_noexceptions.cpp) + set_target_properties(readme_examples_noexceptions11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF) + + # Compile tests that *should fail* + add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp) + target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0) +endif() + + +if(MSVC AND BUILD_SHARED_LIBS) + add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event + COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..." + "$" # <--this is in-file + "$") # <--this is out-file path +endif() + +if(NOT MSVC) + ###### + # This tests is to guard us against ever again removing the soversion + # number from the library. See Bug + # https://github.com/simdjson/simdjson/issues/1014 + ##### + get_target_property(REPORTED_SIMDJSON_SOVERSION simdjson SOVERSION) + if(NOT ${REPORTED_SIMDJSON_SOVERSION} STREQUAL ${SIMDJSON_LIB_SOVERSION}) + message(FATAL_ERROR "The library target does not have the proper soversion information." ) + endif() + get_target_property(REPORTED_SIMDJSON_VERSION simdjson VERSION) + if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION}) + message(FATAL_ERROR "The library target does not have the proper version information." ) + endif() +endif() \ No newline at end of file diff --git a/tests/allparserscheckfile.cpp b/tests/dom/allparserscheckfile.cpp similarity index 100% rename from tests/allparserscheckfile.cpp rename to tests/dom/allparserscheckfile.cpp diff --git a/tests/basictests.cpp b/tests/dom/basictests.cpp similarity index 99% rename from tests/basictests.cpp rename to tests/dom/basictests.cpp index c3877afab..6e4228e18 100644 --- a/tests/basictests.cpp +++ b/tests/dom/basictests.cpp @@ -687,7 +687,7 @@ namespace dom_api_tests { return false; } if (!iter.down()) { - printf("Root should not be emtpy\n"); + printf("Root should not be empty\n"); return false; } if (!iter.is_string()) { @@ -708,7 +708,7 @@ namespace dom_api_tests { return false; } if(!iter.down()) { - printf("Image key should not be emtpy\n"); + printf("Image key should not be empty\n"); return false; } if(!iter.next()) { diff --git a/tests/document_stream_tests.cpp b/tests/dom/document_stream_tests.cpp similarity index 100% rename from tests/document_stream_tests.cpp rename to tests/dom/document_stream_tests.cpp diff --git a/tests/document_tests.cpp b/tests/dom/document_tests.cpp similarity index 98% rename from tests/document_tests.cpp rename to tests/dom/document_tests.cpp index 0023ee389..8ee4122b3 100644 --- a/tests/document_tests.cpp +++ b/tests/dom/document_tests.cpp @@ -36,7 +36,7 @@ namespace document_tests { return true; } - // adversarial example that once triggred overruns, see https://github.com/lemire/simdjson/issues/345 + // adversarial example that once triggered overruns, see https://github.com/simdjson/simdjson/issues/345 bool bad_example() { std::cout << __func__ << std::endl; simdjson::padded_string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"_padded; diff --git a/tests/errortests.cpp b/tests/dom/errortests.cpp similarity index 100% rename from tests/errortests.cpp rename to tests/dom/errortests.cpp diff --git a/tests/extracting_values_example.cpp b/tests/dom/extracting_values_example.cpp similarity index 100% rename from tests/extracting_values_example.cpp rename to tests/dom/extracting_values_example.cpp diff --git a/tests/integer_tests.cpp b/tests/dom/integer_tests.cpp similarity index 100% rename from tests/integer_tests.cpp rename to tests/dom/integer_tests.cpp diff --git a/tests/issue150.sh b/tests/dom/issue150.sh similarity index 100% rename from tests/issue150.sh rename to tests/dom/issue150.sh diff --git a/tests/jsoncheck.cpp b/tests/dom/jsoncheck.cpp similarity index 100% rename from tests/jsoncheck.cpp rename to tests/dom/jsoncheck.cpp diff --git a/tests/minefieldcheck.cpp b/tests/dom/minefieldcheck.cpp similarity index 100% rename from tests/minefieldcheck.cpp rename to tests/dom/minefieldcheck.cpp diff --git a/tests/numberparsingcheck.cpp b/tests/dom/numberparsingcheck.cpp similarity index 100% rename from tests/numberparsingcheck.cpp rename to tests/dom/numberparsingcheck.cpp diff --git a/tests/parse_many_test.cpp b/tests/dom/parse_many_test.cpp similarity index 99% rename from tests/parse_many_test.cpp rename to tests/dom/parse_many_test.cpp index 291b7cde7..96affa009 100644 --- a/tests/parse_many_test.cpp +++ b/tests/dom/parse_many_test.cpp @@ -58,7 +58,7 @@ bool validate(const char *dirname) { bool everything_fine = true; const char *extension1 = ".ndjson"; const char *extension2 = ".jsonl"; - const char *extension3 = ".json"; // bad json files shoud fail + const char *extension3 = ".json"; // bad json files should fail size_t dirlen = std::strlen(dirname); struct dirent **entry_list; diff --git a/tests/pointercheck.cpp b/tests/dom/pointercheck.cpp similarity index 100% rename from tests/pointercheck.cpp rename to tests/dom/pointercheck.cpp diff --git a/tests/random_string_number_tests.cpp b/tests/dom/random_string_number_tests.cpp similarity index 100% rename from tests/random_string_number_tests.cpp rename to tests/dom/random_string_number_tests.cpp diff --git a/tests/readme_examples.cpp b/tests/dom/readme_examples.cpp similarity index 100% rename from tests/readme_examples.cpp rename to tests/dom/readme_examples.cpp diff --git a/tests/readme_examples_noexceptions.cpp b/tests/dom/readme_examples_noexceptions.cpp similarity index 100% rename from tests/readme_examples_noexceptions.cpp rename to tests/dom/readme_examples_noexceptions.cpp diff --git a/tests/singleheadertest.cpp b/tests/dom/singleheadertest.cpp similarity index 100% rename from tests/singleheadertest.cpp rename to tests/dom/singleheadertest.cpp diff --git a/tests/stringparsingcheck.cpp b/tests/dom/stringparsingcheck.cpp similarity index 100% rename from tests/stringparsingcheck.cpp rename to tests/dom/stringparsingcheck.cpp diff --git a/tests/testjson2json.sh b/tests/dom/testjson2json.sh similarity index 100% rename from tests/testjson2json.sh rename to tests/dom/testjson2json.sh diff --git a/tests/trivially_copyable_test.cpp b/tests/dom/trivially_copyable_test.cpp similarity index 100% rename from tests/trivially_copyable_test.cpp rename to tests/dom/trivially_copyable_test.cpp diff --git a/tests/ondemand/CMakeLists.txt b/tests/ondemand/CMakeLists.txt index 9f985b4b2..6d7e01ddd 100644 --- a/tests/ondemand/CMakeLists.txt +++ b/tests/ondemand/CMakeLists.txt @@ -26,9 +26,9 @@ if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which endif() # Copy the simdjson dll into the tests directory -if(MSVC) +if(MSVC AND BUILD_SHARED_LIBS) add_custom_command(TARGET ondemand_parse_api_tests POST_BUILD # Adds a post-build event COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..." "$" # <--this is in-file "$") # <--this is out-file path -endif(MSVC) +endif(MSVC AND BUILD_SHARED_LIBS) diff --git a/tests/ondemand/compilation_failure_tests/CMakeLists.txt b/tests/ondemand/compilation_failure_tests/CMakeLists.txt index 2d0959be2..c766be0ad 100644 --- a/tests/ondemand/compilation_failure_tests/CMakeLists.txt +++ b/tests/ondemand/compilation_failure_tests/CMakeLists.txt @@ -8,7 +8,7 @@ # adds a compilation test. Two targets are created, one expected to # succeed compilation and one that is expected to fail. function(add_dual_compile_test TEST_NAME) - add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY LABELS onedemand no_mingw) + add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY LABELS ondemand no_mingw) add_cpp_test(${TEST_NAME}_should_not_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY WILL_FAIL LABELS ondemand acceptance no_mingw) target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1) endfunction(add_dual_compile_test) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 50b0213eb..f3eaa0541 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,5 +1,5 @@ if(TARGET cxxopts) # we only build the tools if cxxopts is available - message(STATUS "We have cxxopts as a dependency and we are buiding the tools (e.g., json2json).") + message(STATUS "We have cxxopts as a dependency and we are building the tools (e.g., json2json).") foreach(tool IN ITEMS json2json jsonstats jsonpointer minify) add_executable("${tool}" "${tool}.cpp") simdjson_apply_props("${tool}") diff --git a/tools/release.py b/tools/release.py index 63c586a70..c3122b1d4 100755 --- a/tools/release.py +++ b/tools/release.py @@ -145,7 +145,7 @@ if(atleastminor): sonumber += 1 for line in fileinput.input(cmakefile, inplace=1, backup='.bak'): - line = re.sub(' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line) + line = re.sub(' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line.rstrip()) line = re.sub('SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line) line = re.sub('set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line) print(line)