admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into jkeiser/no-padding

Browse Source
This commit is contained in:
Daniel Lemire
2021-05-07 23:03:39 -04:00
parent d7b08c84c7 d539781cf3
commit 76f7e4ae04
77 changed files with 409 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.appveyor.yml
+4 -4
View File
@@ -18,19 +18,19 @@ environment:
CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
- job_name: VS2017 (Static, No Threads)
image: Visual Studio 2017
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
CTEST_ARGS: -LE explicitonly
- job_name: VS2019 (Win32)
platform: Win32
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
CTEST_ARGS: -LE explicitonly
- job_name: VS2019 (Win32, No Exceptions)
platform: Win32
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
CTEST_ARGS: -LE explicitonly
- job_name: VS2015
image: Visual Studio 2015
CMAKE_ARGS: -A %Platform% -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_ENABLE_THREADS=OFF
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
CTEST_ARGS: -LE explicitonly
build_script:
.circleci/config.yml
+13 -13
View File
@@ -164,49 +164,49 @@ jobs:
gcc10-perftest:
description: Build and run performance tests on GCC 10 and AVX 2 with a cmake static build, this test performance regression
executor: gcc10
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_BUILD_STATIC=ON }
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DBUILD_SHARED_LIBS=OFF }
steps: [ cmake_perftest ]
gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake static build
executor: gcc10
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON }
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF }
steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ]
clang6:
description: Build and run tests on clang 6 and AVX 2 with a cmake static build
executor: clang6
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON }
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF }
steps: [ cmake_test, cmake_install_test ]
clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake static build
executor: clang10
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON }
environment: { CMAKE_FLAGS: -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF }
steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ]
# libcpp
libcpp-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake static build and libc++
executor: clang10
environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON -DSIMDJSON_BUILD_STATIC=ON }
environment: { CMAKE_FLAGS: -DSIMDJSON_USE_LIBCPP=ON -DBUILD_SHARED_LIBS=OFF }
steps: [ cmake_test, cmake_install_test, cmake_installed_test_cxx20 ]
# sanitize
sanitize-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
executor: gcc10
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
sanitize-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
executor: clang10
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
threadsanitize-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
executor: gcc10
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
threadsanitize-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
executor: clang10
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
nocheckeof-clang10:
description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
@@ -220,12 +220,12 @@ jobs:
dynamic-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake dynamic build
executor: gcc10
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON }
steps: [ cmake_test, cmake_install_test ]
dynamic-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake dynamic build
executor: clang10
environment: { CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF }
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON }
steps: [ cmake_test, cmake_install_test ]
# unthreaded
@@ -270,12 +270,12 @@ jobs:
sanitize-haswell-gcc10:
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
executor: gcc10
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
sanitize-haswell-clang10:
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
executor: clang10
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
steps: [ cmake_test ]
workflows:
.drone.yml
+37 -35
View File
@@ -8,7 +8,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
@@ -29,7 +29,7 @@ steps:
CC: clang-6.0
CXX: clang++-6.0
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
@@ -50,7 +50,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -77,7 +77,7 @@ steps:
CC: clang-6.0
CXX: clang++-6.0
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- mkdir build
@@ -100,7 +100,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -122,7 +122,7 @@ steps:
environment:
CC: clang-9
CXX: clang++-9
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -142,7 +142,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -213,7 +213,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -237,7 +237,7 @@ steps:
environment:
CC: clang-6.0
CXX: clang++-6.0
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -260,7 +260,7 @@ steps:
CC: gcc
CXX: g++
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
@@ -281,7 +281,7 @@ steps:
environment:
CC: clang-6.0
CXX: clang++-6.0
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=OFF
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
BUILD_FLAGS: -- -j
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
commands:
@@ -302,7 +302,7 @@ steps:
image: gcc:8
environment:
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CC: gcc
CXX: g++
@@ -356,7 +356,7 @@ steps:
CC: clang-9
CXX: clang++-9
BUILD_FLAGS: -- -j 4
CMAKE_FLAGS: -GNinja -DSIMDJSON_BUILD_STATIC=ON
CMAKE_FLAGS: -GNinja -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -stdlib=libc++
commands:
@@ -377,7 +377,7 @@ steps:
CC: clang-9
CXX: clang++-9
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -stdlib=libc++
commands:
@@ -398,7 +398,7 @@ steps:
CC: clang-7
CXX: clang++-7
BUILD_FLAGS: -- -j
CMAKE_FLAGS: -DSIMDJSON_BUILD_STATIC=ON
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
CXXFLAGS: -stdlib=libc++
commands:
@@ -429,26 +429,28 @@ steps:
- cmake $CMAKE_FLAGS ..
- cmake --build . $BUILD_FLAGS
- ctest $CTEST_FLAGS
---
kind: pipeline
name: arm64-fuzz
platform: { os: linux, arch: arm64 }
steps:
- name: Build and run fuzzers shortly
image: ubuntu:20.04
environment:
CC: clang
CXX: clang++
DEBIAN_FRONTEND: noninteractive
ASAN_OPTIONS: detect_leaks=0
commands:
- apt-get -qq update
- apt-get install -q -y clang cmake git wget zip ninja-build
- wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar
- tar xf corpus.tar && rm corpus.tar
- fuzz/build_like_ossfuzz.sh
- mkdir -p common_out
- for fuzzer in build/fuzz/fuzz_* ; do echo $fuzzer;$fuzzer common_out out/* -max_total_time=40; done
##############################
# With bintray gone, it is clear how we should fuzz.
#---
#kind: pipeline
#name: arm64-fuzz
#platform: { os: linux, arch: arm64 }
#steps:
#- name: Build and run fuzzers shortly
# image: ubuntu:20.04
# environment:
# CC: clang
# CXX: clang++
# DEBIAN_FRONTEND: noninteractive
# ASAN_OPTIONS: detect_leaks=0
# commands:
# - apt-get -qq update
# - apt-get install -q -y clang cmake git wget zip ninja-build
# - wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar
# - tar xf corpus.tar && rm corpus.tar
# - fuzz/build_like_ossfuzz.sh
# - mkdir -p common_out
# - for fuzzer in build/fuzz/fuzz_* ; do echo $fuzzer;$fuzzer common_out out/* -max_total_time=40; done
---
kind: pipeline
name: stylecheck
.github/workflows/mingw-ci.yml
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
runs-on: windows-2016
env:
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
CC: gcc
CXX: g++
@@ -63,6 +63,6 @@ jobs:
git --version
mkdir build32
cd build32
cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
.github/workflows/mingw64-ci.yml
+3 -3
View File
@@ -22,7 +22,7 @@ jobs:
runs-on: windows-2016
env:
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DSIMDJSON_BUILD_STATIC=ON .. ' if using the command line
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
CC: gcc
CXX: g++
@@ -63,12 +63,12 @@ jobs:
git --version
mkdir build64
cd build64
cmake -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
cd ..
mkdir build64debug
cd build64debug
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
cmake --build . --target acceptance_tests --verbose
ctest -L acceptance --output-on-failure
.github/workflows/msys2.yml
+1 -1
View File
@@ -52,6 +52,6 @@ jobs:
run: |
mkdir build
cd build
cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
cmake --build . --verbose
ctest -j4 --output-on-failure -LE explicitonly
.github/workflows/power-fuzz.yml
+13 -4
View File
@@ -16,6 +16,16 @@ jobs:
name: Build on ubuntu-20.04 ppc64le
steps:
- uses: actions/checkout@v2.1.0
- uses: actions/cache@v2
id: cache-corpus
with:
path: out/
key: corpus-${{ github.run_id }}
restore-keys: corpus-
- name: show statistics for the cached corpus
run: |
echo number of files in github action corpus cache:
find out -type f |wc -l
- uses: uraimo/run-on-arch-action@v2.0.5
name: Run commands
id: runcmd
@@ -42,8 +52,9 @@ jobs:
cmake .. -GNinja \
-DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX \
-DCMAKE_C_COMPILER=clang$CLANGSUFFIX \
-DSIMDJSON_BUILD_STATIC=Off \
-DENABLE_FUZZING=On \
-DBUILD_SHARED_LIBS=OFF \
-DSIMDJSON_DEVELOPER_MODE=ON \
-DSIMDJSON_ENABLE_FUZZING=On \
-DSIMDJSON_COMPETITION=OFF \
-DSIMDJSON_GOOGLE_BENCHMARKS=OFF \
-DSIMDJSON_DISABLE_DEPRECATED_API=On \
@@ -55,8 +66,6 @@ jobs:
cd ..
builddir=build
cmake --build $builddir
wget --quiet https://dl.bintray.com/pauldreik/simdjson-fuzz-corpus/corpus/corpus.tar
tar xf corpus.tar && rm corpus.tar
fuzzernames=$(cmake --build $builddir --target print_all_fuzzernames |tail -n1)
for fuzzer in $fuzzernames ; do
exe=$builddir/fuzz/$fuzzer
.github/workflows/ubuntu18-checkperf.yml
+1 -1
View File
@@ -24,6 +24,6 @@ jobs:
run: |
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . --target checkperf &&
ctest --output-on-failure -R checkperf ubuntu18-checkperf.yml
.github/workflows/ubuntu18.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
run: |
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
make install &&
.github/workflows/ubuntu20-checkperf.yml
+1 -1
View File
@@ -24,6 +24,6 @@ jobs:
run: |
mkdir build &&
cd build &&
cmake -DCMAKE_CXX_FLAGS="-Werror=old-style-cast -pedantic -Wpedantic" -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake -DCMAKE_CXX_FLAGS="-Werror=old-style-cast -pedantic -Wpedantic" -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . --target checkperf &&
ctest --output-on-failure -R checkperf
.github/workflows/ubuntu20-noexcept.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
run: |
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_EXCEPTIONS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
make install &&
.github/workflows/ubuntu20-nothread.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
run: |
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_ENABLE_THREADS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
make install &&
.github/workflows/ubuntu20.yml
+1 -1
View File
@@ -24,7 +24,7 @@ jobs:
run: |
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest -j --output-on-failure -LE explicitonly &&
cmake --install . &&
.github/workflows/vs16-clang-ci.yml
+1 -1
View File
@@ -29,7 +29,7 @@ jobs:
buildDirectory: "${{ github.workspace }}/../../_temp/windows"
cmakeBuildType: Release
buildWithCMake: true
cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=ON
cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=OFF
buildWithCMakeArgs: --config Release
- name: 'Run CTest'
.github/workflows/vs16-ninja-ci.yml
+1 -1
View File
@@ -29,7 +29,7 @@ jobs:
buildDirectory: "${{ github.workspace }}/../../_temp/windows"
cmakeBuildType: Release
buildWithCMake: true
cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=ON
cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=OFF
buildWithCMakeArgs: --config Release
- name: 'Run CTest'
run: ctest -C Release -LE explicitonly --output-on-failure
.travis.yml
+1 -1
View File
@@ -176,7 +176,7 @@ install:
export ASAN_OPTIONS="detect_leaks=0";
fi
- if [[ "${STATIC}" == "on" ]]; then
export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_BUILD_STATIC=ON";
export CMAKE_FLAGS="${CMAKE_FLAGS} -DBUILD_SHARED_LIBS=OFF";
fi
- export CTEST_FLAGS="-j4 --output-on-failure -LE explicitonly"
CONTRIBUTING.md
+1 -1
View File
@@ -69,7 +69,7 @@ Pull requests are always invited. However, we ask that you follow these guidelin
3. Other types of changes must be clearly motivated. We openly discourage changes with no identifiable benefits.
- Changes should be focused and minimal. You should change as few lines of code as possible. Please do not reformat or touch files needlessly.
- New features must be accompanied of new tests, in general.
- Your code should pass our continuous-integration tests. It is your responsability to ensure that your proposal pass the tests. We do not merge pull requests that would break our build.
- Your code should pass our continuous-integration tests. It is your responsibility to ensure that your proposal pass the tests. We do not merge pull requests that would break our build.
- An exception to this would be changes to non-code files, such as documentation and assets, or trivial changes to code, such as comments, where it is encouraged to explicitly ask for skipping a CI run using the `[skip ci]` prefix in your Pull Request title **and** in the first line of the most recent commit in a push. Example for such a commit: `[skip ci] Fixed typo in power_of_ten's docs`
This benefits the project in such a way that the CI pipeline is not burdened by running jobs on changes that don't change any behavior in the code, which reduces wait times for other Pull Requests that do change behavior and require testing.
CONTRIBUTORS
+1
View File
@@ -36,5 +36,6 @@ Nong Li
Furkan Taşkale
Brendan Knapp
Danila Kutenin
Pavel Pavlov
# if you have contributed to the project and your name does not
# appear in this list, please let us know!
Dockerfile
-88
View File
@@ -1,88 +0,0 @@
###
#
# Though simdjson requires only commonly available compilers and tools, it can
# be convenient to build it and test it inside a docker container: it makes it
# possible to test and benchmark simdjson under even relatively out-of-date
# Linux servers. It should also work under macOS and Windows, though not
# at native speeds, maybe.
#
# Assuming that you have a working docker server, this file
# allows you to build, test and benchmark simdjson.
#
# We build the library and associated files in the dockerbuild subdirectory.
# It may be necessary to delete it before creating the image:
#
# rm -r -f dockerbuild
#
# The need to delete the directory has nothing to do with docker per se: it is
# simply cleaner in CMake to start from a fresh directory. This is important: if you
# reuse the same directory with different configurations, you may get broken builds.
#
#
# Then you can build the image as follows:
#
# docker build -t simdjson --build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g) .
#
# Please note that the image does not contain a copy of the code. However, the image will contain the
# the compiler and the build system. This means that if you change the source code, after you have built
# the image, you won't need to rebuild the image. In fact, unless you want to try a different compiler, you
# do not need to ever rebuild the image, even if you do a lot of work on the source code.
#
# We specify the users to avoid having files owned by a privileged user (root) in our directory. Some
# people like to run their machine as the "root" user. We do not think it is cool.
#
# Then you need to build the project:
#
# docker run -v $(pwd):/project:Z simdjson
#
# Should you change a source file, you may need to call this command again. Because the output
# files are persistent between calls to this command (they reside in the dockerbuild directory),
# this command can be fast.
#
# Next you can test it as follows:
#
# docker run -it -v $(pwd):/project:Z simdjson sh -c "cd dockerbuild && ctest . --output-on-failure -LE explicitonly"
#
# The run the complete tests requires you to have built all of simdjson.
#
# Building all of simdjson takes a long time. Instead, you can build just one target:
#
# docker run -it -v $(pwd):/project:Z simdjson sh -c "[ -d dockerbuild ] || mkdir dockerbuild && cd dockerbuild && cmake .. && cmake --build . --target parse"
#
# Note that it is safe to remove dockerbuild before call the previous command, as the repository gets rebuild. It is also possible, by changing the command, to use a different directory name.
#
# You can run performance tests:
#
# docker run -it --privileged -v $(pwd):/project:Z simdjson sh -c "cd dockerbuild && for i in ../jsonexamples/*.json; do echo \$i; ./benchmark/parse \$i; done"
#
# The "--privileged" is recommended so you can get performance counters under Linux.
#
# You can also grab a fresh copy of simdjson and rebuild it, to make comparisons:
#
# docker run -it -v $(pwd):/project:Z simdjson sh -c "git clone https://github.com/simdjson/simdjson.git && cd simdjson && mkdir build && cd build && cmake .. && cmake --build . --target parse "
#
# Then you can run comparisons:
#
# docker run -it --privileged -v $(pwd):/project:Z simdjson sh -c "for i in jsonexamples/*.json; do echo \$i; dockerbuild/benchmark/parse \$i| grep GB| head -n 1; simdjson/build/benchmark/parse \$i | grep GB |head -n 1; done"
#
####
FROM ubuntu:20.10
################
# We would prefer to use the conan io images but they do not support 64-bit ARM? The small gcc images appear to
# be broken on ARM.
# Furthermore, we would not expect users to frequently rebuild the container, so using ubuntu is probably fine.
###############
ARG USER_ID
ARG GROUP_ID
RUN apt-get update -qq
RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata
RUN apt-get install -y cmake g++ git
RUN mkdir project
RUN addgroup --gid $GROUP_ID user; exit 0
RUN adduser --disabled-password --gecos '' --uid $USER_ID --gid $GROUP_ID user; exit 0
USER user
RUN gcc --version
WORKDIR /project
CMD ["sh","-c","[ -d dockerbuild ] || mkdir dockerbuild && cd dockerbuild && cmake .. && cmake --build . "]
HACKING.md
+2 -2
View File
@@ -88,7 +88,7 @@ Other important files and directories:
cd build
cmake -D SIMDJSON_DEVELOPER_MODE=ON ..
cmake --build . --config Release
benchmark/parse ../jsonexamples/twitter.json
benchmark/dom/parse ../jsonexamples/twitter.json
```
The last line becomes `./benchmark/Release/parse.exe ../jsonexample/twitter.json` under Windows. You may also use Google Benchmark:
```bash
@@ -159,7 +159,7 @@ processor.
At this point, we are require to use one of two main strategies.
1. On POSIX systems, the main compilers (LLVM clang, GNU gcc) allow us to use any intrinsic function after including the header, but they fail to inline the resulting instruction if the target processor does not support them. Because we compile for a generic processor, we would not be able to use most intrinsic functions. Thankfully, more recent versions of these compilers allow us to flag a region of code with a specific target, so that we can compile only some of the code with support for advanced instructions. Thus in our C++, one might notice macros like `TARGET_HASWELL`. It is then our responsability, at runtime, to only run the regions of code (that we call kernels) matching the properties of the runtime processor. The benefit of this approach is that the compiler not only let us use intrinsic functions, but it can also optimize the rest of the code in the kernel with advanced instructions we enabled.
1. On POSIX systems, the main compilers (LLVM clang, GNU gcc) allow us to use any intrinsic function after including the header, but they fail to inline the resulting instruction if the target processor does not support them. Because we compile for a generic processor, we would not be able to use most intrinsic functions. Thankfully, more recent versions of these compilers allow us to flag a region of code with a specific target, so that we can compile only some of the code with support for advanced instructions. Thus in our C++, one might notice macros like `TARGET_HASWELL`. It is then our responsibility, at runtime, to only run the regions of code (that we call kernels) matching the properties of the runtime processor. The benefit of this approach is that the compiler not only let us use intrinsic functions, but it can also optimize the rest of the code in the kernel with advanced instructions we enabled.
2. Under Visual Studio, the problem is somewhat simpler. Visual Studio will not only provide the intrinsic functions, but it will also allow us to use them. They will compile just fine. It is at runtime that they may cause a crash. So we do not need to mark regions of code for compilation toward advanced processors (e.g., with `TARGET_HASWELL` macros). The downside of the Visual Studio approach is that the compiler is not allowed to use advanced instructions others than those we specify. In principle, this means that Visual Studio has weaker optimization opportunities.
README.md
+2 -3
View File
@@ -1,4 +1,4 @@
[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/simdjson.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&q=proj%3Asimdjson&can=2)
![Ubuntu 18.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2018.04%20CI%20(GCC%207)/badge.svg)
[![Ubuntu 20.04 CI](https://github.com/simdjson/simdjson/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg)](https://simdjson.org/plots.html)
![VS16-CI](https://github.com/simdjson/simdjson/workflows/VS16-CI/badge.svg)
@@ -27,7 +27,6 @@ Table of Contents
-----------------
* [Quick Start](#quick-start)
* [On Demand](#on-demand)
* [Documentation](#documentation)
* [Performance results](#performance-results)
* [Real-world usage](#real-world-usage)
@@ -186,4 +185,4 @@ For compilers that do not support [C++17](https://en.wikipedia.org/wiki/C%2B%2B1
For efficient number serialization, we bundle Florian Loitsch's implementation of the Grisu2 algorithm for binary to decimal floating-point numbers. The implementation was slightly modified by JSON for Modern C++ library. Both Florian Loitsch's implementation and JSON for Modern C++ are provided under the MIT license.
For runtime dispatching, we use some code from the PyTorch project licensed under 3-clause BSD.
For runtime dispatching, we use some code from the PyTorch project licensed under 3-clause BSD.
benchmark/CMakeLists.txt
+3 -12
View File
@@ -1,20 +1,12 @@
add_subdirectory(dom)
include_directories( . linux )
link_libraries(simdjson-windows-headers test-data)
link_libraries(simdjson)
add_executable(benchfeatures benchfeatures.cpp)
add_executable(get_corpus_benchmark get_corpus_benchmark.cpp)
add_executable(perfdiff perfdiff.cpp)
add_executable(parse parse.cpp)
add_executable(parse_stream parse_stream.cpp)
add_executable(statisticalmodel statisticalmodel.cpp)
add_executable(parse_noutf8validation parse.cpp)
target_compile_definitions(parse_noutf8validation PRIVATE SIMDJSON_SKIPUTF8VALIDATION)
add_executable(parse_nonumberparsing parse.cpp)
target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARSING)
add_executable(parse_nostringparsing parse.cpp)
target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING)
if (TARGET benchmark::benchmark)
link_libraries(benchmark::benchmark)
@@ -37,4 +29,3 @@ if (TARGET benchmark::benchmark)
endif()
endif()
include(checkperf.cmake)
benchmark/Dockerfile
-19
View File
@@ -1,19 +0,0 @@
# From the ROOT, run:
# docker build -t simdjsonbench -f benchmark/Dockerfile . && docker run --privileged -t simdjsonbench
FROM gcc:8.3
# # Build latest
# ENV latest_release=v0.2.1
# WORKDIR /usr/src/$latest_release/
# RUN git clone --depth 1 https://github.com/lemire/simdjson/ -b $latest_release .
# RUN make parse
# # Build master
# WORKDIR /usr/src/master/
# RUN git clone --depth 1 https://github.com/lemire/simdjson/ .
# RUN make parse
# Build the current source
COPY . /usr/src/current/
WORKDIR /usr/src/current/
RUN make checkperf
benchmark/dom/CMakeLists.txt
+16
View File
@@ -0,0 +1,16 @@
include_directories( .. ../linux )
link_libraries(simdjson-windows-headers test-data)
link_libraries(simdjson)
add_executable(perfdiff perfdiff.cpp)
add_executable(parse parse.cpp)
add_executable(parse_stream parse_stream.cpp)
add_executable(statisticalmodel statisticalmodel.cpp)
add_executable(parse_noutf8validation parse.cpp)
target_compile_definitions(parse_noutf8validation PRIVATE SIMDJSON_SKIPUTF8VALIDATION)
add_executable(parse_nonumberparsing parse.cpp)
target_compile_definitions(parse_nonumberparsing PRIVATE SIMDJSON_SKIPNUMBERPARSING)
add_executable(parse_nostringparsing parse.cpp)
target_compile_definitions(parse_nostringparsing PRIVATE SIMDJSON_SKIPSTRINGPARSING)
include(checkperf.cmake)
benchmark/checkperf.cmake → benchmark/dom/checkperf.cmake
+1 -1
View File
@@ -76,7 +76,7 @@ if (Git_FOUND AND (GIT_VERSION_STRING VERSION_GREATER "2.1.4") AND (NOT CMAKE_G
if (CMAKE_CONFIGURATION_TYPES)
set(CHECKPERF_PARSE ${SIMDJSON_CHECKPERF_DIR}/build/benchmark/$<CONFIGURATION>/parse)
else()
set(CHECKPERF_PARSE ${SIMDJSON_CHECKPERF_DIR}/build/benchmark/parse)
set(CHECKPERF_PARSE ${SIMDJSON_CHECKPERF_DIR}/build/benchmark/dom/parse)
endif()
add_custom_target(
checkperf-parse ALL # TODO is ALL necessary?
benchmark/parse.cpp → benchmark/dom/parse.cpp
View File
benchmark/parse_stream.cpp → benchmark/dom/parse_stream.cpp
View File
benchmark/perfdiff.cpp → benchmark/dom/perfdiff.cpp
View File
benchmark/statisticalmodel.cpp → benchmark/dom/statisticalmodel.cpp
View File
cmake/exception-flags.cmake
+1 -1
View File
@@ -19,7 +19,7 @@ editing CMAKE_CXX_FLAGS")
# Here, we disable both with the - argument negation operator
string(REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
# Because we cannot change the flag above on an invidual target (yet), the
# Because we cannot change the flag above on an individual target (yet), the
# definition below must similarly be added globally
add_definitions(-D_HAS_EXCEPTIONS=0)
elseif(CMAKE_COMPILER_IS_GNUCC)
doc/basics.md
+2 -2
View File
@@ -459,7 +459,7 @@ select the value. If your keys contain the characters '/' or '~', they must be e
'~0' respectively. An empty JSON Path refers to the whole document.
We also extend the JSON Pointer support to include *relative* paths.
You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document.
You can apply a JSON path to any node and the path gets interpreted relatively, as if the current node were a whole JSON document.
Consider the following example:
@@ -816,7 +816,7 @@ Unlike `parser.parse`, both `parser.load_many(filename)` and `parser.parse_many(
document at a time.
1. When calling `parser.load_many(filename)`, the file's content is loaded up in a memory buffer owned by the `parser`'s instance. Thus the file can be safely deleted after calling `parser.load_many(filename)` as the parser instance owns all of the data.
2. When calling `parser.parse_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.parse_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents. In constrast, calling `doc = parser.parse("[1,2,3]"_padded)` is safe because `parser.parse` eagerly parses the input.
2. When calling `parser.parse_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.parse_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents. In contrast, calling `doc = parser.parse("[1,2,3]"_padded)` is safe because `parser.parse` eagerly parses the input.
Both `load_many` and `parse_many` take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents.
doc/dom.md
+2 -2
View File
@@ -213,7 +213,7 @@ select the value. If your keys contain the characters '/' or '~', they must be e
'~0' respectively. An empty JSON Path refers to the whole document.
We also extend the JSON Pointer support to include *relative* paths.
You can apply a JSON path to any node and the path gets interpreted relatively, as if the currrent node were a whole JSON document.
You can apply a JSON path to any node and the path gets interpreted relatively, as if the current node were a whole JSON document.
Consider the following example:
@@ -632,4 +632,4 @@ simdjson::dom::parser parser;
simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false);
````
Setting the `realloc_if_needed` parameter `false` in this manner may lead to better performance since copies are avoided, but it requires that the user takes more responsibilities: the simdjson library cannot verify that the input buffer was padded with SIMDJSON_PADDING extra bytes.
Setting the `realloc_if_needed` parameter `false` in this manner may lead to better performance since copies are avoided, but it requires that the user takes more responsibilities: the simdjson library cannot verify that the input buffer was padded with SIMDJSON_PADDING extra bytes.
doc/ondemand_design.md
+1 -1
View File
@@ -636,7 +636,7 @@ direct raw ASCII comparisons: `key().raw()` provides direct access to the unesca
You can compare `key()` with unescaped C strings (e.g., `key()=="test"`). It is expected
that the provided string is a valid JSON string. Importantly,
the C string must not contain an unescaped quote character (`"`). For speed, the comparison is done byte-by-byte
without handling the escaped caracters.
without handling the escaped characters.
If you occasionally need to access and store the
unescaped key values, you may use the `unescaped_key()` method. Once you have called `unescaped_key()` method,
neither the `key()` nor the `unescaped_key()` methods should be called: the current field instance
fuzz/CMakeLists.txt
+5 -5
View File
@@ -1,11 +1,11 @@
if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL)
option(ENABLE_FUZZING "enable building the fuzzers" ON)
option(SIMDJSON_ENABLE_FUZZING "enable building the fuzzers" ON)
else()
option(ENABLE_FUZZING "enable building the fuzzers" OFF)
option(SIMDJSON_ENABLE_FUZZING "enable building the fuzzers" OFF)
endif()
if(ENABLE_FUZZING)
if(SIMDJSON_ENABLE_FUZZING)
# First attempt at a fuzzer, using libFuzzer.
#
@@ -17,7 +17,7 @@ if(ENABLE_FUZZING)
# export CFLAGS="-fsanitize=fuzzer-no-link,address,undefined"
# export CXX=clang++
# export CC=clang++
# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer
# cmake .. -GNinja -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_ENABLE_FUZZING=On -DSIMDJSON_FUZZ_LINKMAIN=Off -DSIMDJSON_FUZZ_LDFLAGS=-fsanitize=fuzzer
# ninja
# settings this links in a main. useful for reproducing,
@@ -25,7 +25,7 @@ if(ENABLE_FUZZING)
# (note that libFuzzer can also reproduce, just pass it the files)
#
# Using this by default, means the fuzzers will be built as a part of the normal
# workflow, meaning they wont bitrot and will participate in refactoring etc.
# workflow, meaning they won't bitrot and will participate in refactoring etc.
#
option(SIMDJSON_FUZZ_LINKMAIN "links a main into fuzz targets for building reproducers" On)
fuzz/build_fuzzer_variants.sh
+1 -1
View File
@@ -48,7 +48,7 @@ if [ -e $testfuzzer ] ; then rm $testfuzzer; fi
# common options
CXX_CLAGS_COMMON=-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_BUILD_STATIC=Off -DENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS"
COMMON="-GNinja -DCMAKE_CXX_COMPILER=clang++$CLANGSUFFIX -DCMAKE_C_COMPILER=clang$CLANGSUFFIX -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_FUZZING=On -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DISABLE_DEPRECATED_API=On -DSIMDJSON_FUZZ_LDFLAGS=$SIMDJSON_FUZZ_LDFLAGS"
# A replay build, as plain as it gets. For use with valgrind/gdb.
variant=replay
fuzz/measure_coverage.sh
+3 -2
View File
@@ -23,8 +23,9 @@ if [ ! -d $bdir ] ; then
cmake .. \
-GNinja \
-DCMAKE_BUILD_TYPE=Debug \
-DSIMDJSON_BUILD_STATIC=On \
-DENABLE_FUZZING=On \
-DSIMDJSON_DEVELOPER_MODE=ON \
-DBUILD_SHARED_LIBS=OFF \
-DSIMDJSON_ENABLE_FUZZING=On \
-DSIMDJSON_DISABLE_DEPRECATED_API=On \
-DSIMDJSON_FUZZ_LINKMAIN=On
ninja all_fuzzers
fuzz/ossfuzz.sh
+3 -2
View File
@@ -26,8 +26,9 @@ cd build
cmake .. \
-GNinja \
-DCMAKE_BUILD_TYPE=Debug \
-DSIMDJSON_BUILD_STATIC=On \
-DENABLE_FUZZING=On \
-DSIMDJSON_DEVELOPER_MODE=ON \
-DBUILD_SHARED_LIBS=OFF \
-DSIMDJSON_ENABLE_FUZZING=On \
-DSIMDJSON_COMPETITION=Off \
-DSIMDJSON_FUZZ_LINKMAIN=Off \
-DSIMDJSON_GOOGLE_BENCHMARKS=Off \
include/simdjson.h
+1 -1
View File
@@ -4,7 +4,7 @@
/**
* @mainpage
*
* Check the [README.md](https://github.com/lemire/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second).
* Check the [README.md](https://github.com/simdjson/simdjson/blob/master/README.md#simdjson--parsing-gigabytes-of-json-per-second).
*
* Sample code. See https://github.com/simdjson/simdjson/blob/master/doc/basics.md for more examples.
include/simdjson/arm64/bitmanipulation.h
+34
View File
@@ -46,6 +46,40 @@ simdjson_really_inline int count_ones(uint64_t input_num) {
return vaddv_u8(vcnt_u8(vcreate_u8(input_num)));
}
#if defined(__GNUC__) // catches clang and gcc
/**
* ARM has a fast 64-bit "bit reversal function" that is handy. However,
* it is not generally available as an intrinsic function under Visual
* Studio (though this might be changing). Even under clang/gcc, we
* apparently need to invoke inline assembly.
*/
/*
* We use SIMDJSON_PREFER_REVERSE_BITS as a hint that algorithms that
* work well with bit reversal may use it.
*/
#define SIMDJSON_PREFER_REVERSE_BITS 1
/* reverse the bits */
simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) {
uint64_t rev_bits;
__asm("rbit %0, %1" : "=r"(rev_bits) : "r"(input_num));
return rev_bits;
}
/**
* Flips bit at index 63 - lz. Thus if you have 'leading_zeroes' leading zeroes,
* then this will set to zero the leading bit. It is possible for leading_zeroes to be
* greating or equal to 63 in which case we trigger undefined behavior, but the output
* of such undefined behavior is never used.
**/
NO_SANITIZE_UNDEFINED
simdjson_really_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) {
return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes);
}
#endif
simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) {
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
*result = value1 + value2;
include/simdjson/arm64/simd.h
+1 -1
View File
@@ -364,7 +364,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x
// Explicit conversion to/from unsigned
//
// Under Visual Studio/ARM64 uint8x16_t and int8x16_t are apparently the same type.
// In theory, we could check this occurence with std::same_as and std::enabled_if but it is C++14
// In theory, we could check this occurrence with std::same_as and std::enabled_if but it is C++14
// and relatively ugly and hard to read.
#ifndef SIMDJSON_REGULAR_VISUAL_STUDIO
simdjson_really_inline explicit simd8(const uint8x16_t other): simd8(vreinterpretq_s8_u8(other)) {}
include/simdjson/common_defs.h
+3 -3
View File
@@ -40,7 +40,7 @@ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
* the input buf should be readable up to buf + SIMDJSON_PADDING
* this is a stopgap; there should be a better description of the
* main loop and its behavior that abstracts over this
* See https://github.com/lemire/simdjson/issues/174
* See https://github.com/simdjson/simdjson/issues/174
*/
constexpr size_t SIMDJSON_PADDING = 32;
@@ -163,7 +163,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
* the regular visual studio or clang under visual
* studio, you still need to handle these issues.
*
* Non-Windows sytems do not have this complexity.
* Non-Windows systems do not have this complexity.
*/
#if SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY
// We set SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY when we build a DLL under Windows.
@@ -217,7 +217,7 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
// now it is safe to trigger the include
#include <string_view> // though the file is there, it does not follow that we got the implementation
#if defined(_LIBCPP_STRING_VIEW)
// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceeded C++17,
// Ah! So we under libc++ which under its Library Fundamentals Technical Specification, which preceded C++17,
// included string_view.
// This means that we have string_view *even though* we may not have C++17.
#define SIMDJSON_HAS_STRING_VIEW
include/simdjson/dom/document-inl.h
+1 -1
View File
@@ -37,7 +37,7 @@ inline error_code document::allocate(size_t capacity) noexcept {
// need a capacity of at least capacity + 1, but it is also possible to do
// worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"
//where capacity + 1 tape elements are
// generated, see issue https://github.com/lemire/simdjson/issues/345
// generated, see issue https://github.com/simdjson/simdjson/issues/345
size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64);
// a document with only zero-length strings... could have capacity/3 string
// and we would need capacity/3 * 5 bytes on the string buffer
include/simdjson/dom/document_stream.h
+1 -1
View File
@@ -123,7 +123,7 @@ public:
using iterator_category = std::input_iterator_tag;
/**
* Default contructor.
* Default constructor.
*/
simdjson_really_inline iterator() noexcept;
/**
include/simdjson/dom/parsedjson_iterator.h
+1 -1
View File
@@ -208,7 +208,7 @@ public:
// throughout return true if we can do the navigation, false
// otherwise
// Withing a given scope (series of nodes at the same depth within either an
// Within a given scope (series of nodes at the same depth within either an
// array or an object), we move forward.
// Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, {
// and [. At the object ({) or at the array ([), you can issue a "down" to
include/simdjson/dom/parser.h
+1 -1
View File
@@ -536,7 +536,7 @@ public:
/**
* @private return an error code corresponding to the last parsing attempt, see
* simdjson.h will return UNITIALIZED if no parsing was attempted
* simdjson.h will return UNINITIALIZED if no parsing was attempted
*/
[[deprecated("Use the result of parser.parse() instead")]]
inline int get_error_code() const noexcept;
include/simdjson/dom/serialization.h
+2 -2
View File
@@ -37,7 +37,7 @@ public:
inline void append(simdjson::dom::element value);
/** Append an array to the builder (to be printed) **/
inline void append(simdjson::dom::array value);
/** Append an objet to the builder (to be printed) **/
/** Append an object to the builder (to be printed) **/
inline void append(simdjson::dom::object value);
/** Reset the builder (so that it would print the empty string) **/
simdjson_really_inline void clear();
@@ -149,7 +149,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<sim
* Print JSON to an output stream.
*
* @param out The output stream.
* @param value The objet.
* @param value The object.
* @throw if there is an error with the underlying output stream. simdjson itself will not throw.
*/
inline std::ostream& operator<<(std::ostream& out, simdjson::dom::object value) {
include/simdjson/generic/numberparsing.h
+4 -4
View File
@@ -161,8 +161,8 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg
// Both i and power_of_five_128[index] have their most significant bit set to 1 which
// implies that the either the most or the second most significant bit of the product
// is 1. We pack values in this manner for efficiency reasons: it maximizes the use
// we make of the product. It also makes it easy to reason aboutthe product: there
// 0 or 1 leading zero in the product.
// we make of the product. It also makes it easy to reason about the product: there
// is 0 or 1 leading zero in the product.
// Unless the least significant 9 bits of the high (64-bit) part of the full
// product are all 1s, then we know that the most significant 55 bits are
@@ -277,7 +277,7 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg
mantissa &= ~(1ULL << 52);
// we have to check that real_exponent is in range, otherwise we bail out
if (simdjson_unlikely(real_exponent > 2046)) {
// We have an infinte value!!! We could actually throw an error here if we could.
// We have an infinite value!!! We could actually throw an error here if we could.
return false;
}
d = to_double(mantissa, real_exponent, negative);
@@ -454,7 +454,7 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
// we could extend our code by using a 128-bit integer instead
// of a 64-bit integer. However, this is uncommon in practice.
//
// 9999999999999999999 < 2**64 so we can accomodate 19 digits.
// 9999999999999999999 < 2**64 so we can accommodate 19 digits.
// If we have a decimal separator, then digit_count - 1 is the number of digits, but we
// may not have a decimal separator!
if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) {
include/simdjson/generic/ondemand/raw_json_string.h
+2 -2
View File
@@ -75,7 +75,7 @@ public:
* long strings.
*
* If target is a compile-time constant, and your compiler likes you,
* you should be able to do the following without performance penatly...
* you should be able to do the following without performance penalty...
*
* static_assert(raw_json_string::is_free_from_unescaped_quote(target), "");
* s.unsafe_is_equal(target);
@@ -89,7 +89,7 @@ public:
* the caller is responsible for this check. See is_free_from_unescaped_quote.
*
* If target is a compile-time constant, and your compiler likes you,
* you should be able to do the following without performance penatly...
* you should be able to do the following without performance penalty...
*
* static_assert(raw_json_string::is_free_from_unescaped_quote(target), "");
* s.unsafe_is_equal(target);
include/simdjson/generic/ondemand/serialization.h
+2 -2
View File
@@ -16,7 +16,7 @@ public:
inline simdjson::error_code append(value element) noexcept;
/** Append an array to the builder (to be printed) **/
inline simdjson::error_code append(array value) noexcept;
/** Append an objet to the builder (to be printed) **/
/** Append an object to the builder (to be printed) **/
inline simdjson::error_code append(object value) noexcept;
/** Append a field to the builder (to be printed) **/
inline simdjson::error_code append(field value) noexcept;
@@ -134,7 +134,7 @@ inline std::ostream& operator<<(std::ostream& out, document& value) {
* Print JSON to an output stream.
*
* @param out The output stream.
* @param value The objet.
* @param value The object.
* @throw if there is an error with the underlying output stream. simdjson itself will not throw.
*/
#if SIMDJSON_EXCEPTIONS
jsonexamples/small/jsoniter_scala/README.md
+1 -1
View File
@@ -1,7 +1,7 @@
Files from https://github.com/plokhotnyuk/jsoniter-scala/tree/master/jsoniter-scala-benchmark/src/main/resources/com/github/plokhotnyuk/jsoniter_scala/benchmark
See issue:
https://github.com/lemire/simdjson/issues/70
https://github.com/simdjson/simdjson/issues/70
The files che-*.geo.json are number-parsing stress tests.
src/generic/stage1/json_structural_indexer.h
+56 -1
View File
@@ -31,8 +31,62 @@ public:
// it helps tremendously.
if (bits == 0)
return;
int cnt = static_cast<int>(count_ones(bits));
#if defined(SIMDJSON_PREFER_REVERSE_BITS)
/**
* ARM lacks a fast trailing zero instruction, but it has a fast
* bit reversal instruction and a fast leading zero instruction.
* Thus it may be profitable to reverse the bits (once) and then
* to rely on a sequence of instructions that call the leading
* zero instruction.
*
* Performance notes:
* The chosen routine is not optimal in terms of data dependency
* since zero_leading_bit might require two instructions. However,
* it tends to minimize the total number of instructions which is
* beneficial.
*/
uint64_t rev_bits = reverse_bits(bits);
int cnt = static_cast<int>(count_ones(bits));
int i = 0;
// Do the first 8 all together
for (; i<8; i++) {
int lz = leading_zeroes(rev_bits);
this->tail[i] = static_cast<uint32_t>(idx) + lz;
rev_bits = zero_leading_bit(rev_bits, lz);
}
// Do the next 8 all together (we hope in most cases it won't happen at all
// and the branch is easily predicted).
if (simdjson_unlikely(cnt > 8)) {
i = 8;
for (; i<16; i++) {
int lz = leading_zeroes(rev_bits);
this->tail[i] = static_cast<uint32_t>(idx) + lz;
rev_bits = zero_leading_bit(rev_bits, lz);
}
// Most files don't have 16+ structurals per block, so we take several basically guaranteed
// branch mispredictions here. 16+ structurals per block means either punctuation ({} [] , :)
// or the start of a value ("abc" true 123) every four characters.
if (simdjson_unlikely(cnt > 16)) {
i = 16;
while (rev_bits != 0) {
int lz = leading_zeroes(rev_bits);
this->tail[i++] = static_cast<uint32_t>(idx) + lz;
rev_bits = zero_leading_bit(rev_bits, lz);
}
}
}
this->tail += cnt;
#else // SIMDJSON_PREFER_REVERSE_BITS
/**
* Under recent x64 systems, we often have both a fast trailing zero
* instruction and a fast 'clear-lower-bit' instruction so the following
* algorithm can be competitive.
*/
int cnt = static_cast<int>(count_ones(bits));
// Do the first 8 all together
for (int i=0; i<8; i++) {
this->tail[i] = idx + trailing_zeroes(bits);
@@ -61,6 +115,7 @@ public:
}
this->tail += cnt;
#endif
}
};
tests/CMakeLists.txt
+9 -171
View File
@@ -1,189 +1,27 @@
# Most tests need test data, and many need windows headers.
link_libraries(simdjson-internal-flags test-data simdjson-windows-headers)
include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake)
add_subdirectory(dom)
add_subdirectory(ondemand)
#
# These tests explicitly do #include "simdjson.cpp" so they can override stuff
#
if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL)
add_cpp_test(numberparsingcheck LABELS acceptance per_implementation)
simdjson_apply_props(numberparsingcheck)
target_link_libraries(numberparsingcheck PRIVATE simdjson-windows-headers)
target_compile_definitions(numberparsingcheck PRIVATE NOMINMAX)
target_include_directories(
numberparsingcheck
PRIVATE "${PROJECT_SOURCE_DIR}/src"
)
add_cpp_test(stringparsingcheck LABELS acceptance per_implementation)
simdjson_apply_props(numberparsingcheck)
target_link_libraries(stringparsingcheck PRIVATE simdjson-windows-headers)
target_compile_definitions(stringparsingcheck PRIVATE NOMINMAX)
target_include_directories(
stringparsingcheck
PRIVATE "${PROJECT_SOURCE_DIR}/src"
)
endif()
# All remaining tests link with simdjson proper
link_libraries(simdjson)
if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL)
add_cpp_test(random_string_number_tests LABELS dom acceptance per_implementation)
endif()
add_cpp_test(basictests LABELS dom acceptance per_implementation)
add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation)
add_cpp_test(document_tests LABELS dom acceptance per_implementation)
add_cpp_test(errortests LABELS dom acceptance per_implementation)
add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation)
add_cpp_test(integer_tests LABELS dom acceptance per_implementation)
add_cpp_test(jsoncheck LABELS dom acceptance per_implementation)
add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation)
add_cpp_test(parse_many_test LABELS dom acceptance per_implementation)
add_cpp_test(pointercheck LABELS dom acceptance per_implementation) # https://tools.ietf.org/html/rfc6901
add_cpp_test(unicode_tests LABELS dom acceptance per_implementation)
add_cpp_test(trivially_copyable_test LABELS dom acceptance per_implementation)
add_cpp_test(unicode_tests LABELS dom acceptance per_implementation)
add_cpp_test(minify_tests LABELS other acceptance per_implementation)
add_cpp_test(padded_string_tests LABELS other acceptance )
find_program(BASH bash)
# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to
# map app paths to their "mounted" equivalent (e.g., /mnt/c/...). So even if you have bash under Windows, extra work would be
# required to make things work robustly. Simply put: bash is not quite portable.
# Script tests
if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts are not robust enough to run under Windows even if bash is available
#
# json2json test
#
add_test(
NAME testjson2json
COMMAND ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh
WORKING_DIRECTORY $<TARGET_FILE_DIR:minify>
)
set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json)
set_property(TEST testjson2json APPEND PROPERTY LABELS per_implementation)
add_dependencies(per_implementation_tests minify)
#
# Competition parse test
#
if ((SIMDJSON_COMPETITION) AND (!SIMDJSON_SANITIZE))
# It looks like RapidJSON does not pass the sanitizer under some conditions (Clang 10)
add_executable(allparserscheckfile allparserscheckfile.cpp)
add_dependencies(competition_tests allparserscheckfile)
add_dependencies(per_implementation_tests allparserscheckfile)
target_link_libraries(allparserscheckfile PRIVATE competition-all)
add_test(issue150 ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation competition)
add_dependencies(all_tests allparserscheckfile)
endif()
#
# json2json tool test: check that json2json can parse twitter.json
#
# This tests validates that the implementation is what we think it is if we get passed
# SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing
add_cpp_test(checkimplementation LABELS other per_implementation)
add_test(NAME json2json COMMAND $<TARGET_FILE:json2json> ${EXAMPLE_JSON})
set_property(TEST json2json APPEND PROPERTY LABELS acceptance per_implementation)
add_dependencies(acceptance_tests json2json)
add_dependencies(per_implementation_tests json2json)
#
# SIMDJSON_FORCE_IMPLEMENTATION tests: run json2json with SIMDJSON
#
if (SIMDJSON_IMPLEMENTATION MATCHES "fallback")
add_test(
NAME simdjson_force_implementation
COMMAND
${CMAKE_COMMAND} -E env
SIMDJSON_FORCE_IMPLEMENTATION=fallback
$<TARGET_FILE:checkimplementation>
)
endif()
add_test(
NAME simdjson_force_implementation_error
COMMAND
${CMAKE_COMMAND} -E env
SIMDJSON_FORCE_IMPLEMENTATION=doesnotexist
$<TARGET_FILE:json2json> ${EXAMPLE_JSON}
)
set_tests_properties(simdjson_force_implementation_error PROPERTIES WILL_FAIL TRUE)
endif()
#
# Compile-only tests with simdjson flags on
#
# Don't add the tests if we're on VS2017 or older; they don't succeed.
if(NOT (MSVC AND MSVC_VERSION LESS 1920))
if(SIMDJSON_EXCEPTIONS)
add_cpp_test(readme_examples COMPILE_ONLY LABELS acceptance)
add_cpp_test(readme_examples11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
set_target_properties(readme_examples11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
endif()
add_cpp_test(readme_examples_noexceptions COMPILE_ONLY LABELS acceptance)
add_cpp_test(readme_examples_noexceptions11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples_noexceptions.cpp)
set_target_properties(readme_examples_noexceptions11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
# Compile tests that *should fail*
add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0)
endif()
if(MSVC)
add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E echo "$<TARGET_FILE:simdjson>"
COMMAND ${CMAKE_COMMAND} -E echo "$<TARGET_FILE_DIR:basictests>"
if(MSVC AND BUILD_SHARED_LIBS)
# Copy the simdjson dll into the tests directory
add_custom_command(TARGET unicode_tests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
"$<TARGET_FILE_DIR:basictests>") # <--this is out-file path
"$<TARGET_FILE_DIR:unicode_tests>") # <--this is out-file path
endif()
# Copy the simdjson dll into the tests directory
if(MSVC)
add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
"$<TARGET_FILE_DIR:basictests>") # <--this is out-file path
endif()
## Next bit should not be needed!
#if(CMAKE_INTERPROCEDURAL_OPTIMIZATION)
# next line is a workaround for an odr-violation in basictests regarding the globals 0x432a40 and 0x52045c under clang
#set_tests_properties(basictests PROPERTIES
# ENVIRONMENT ASAN_OPTIONS="detect_odr_violation=0")
#endif()
## This causes problems
# add_executable(singleheader ./singleheadertest.cpp ${PROJECT_SOURCE_DIR}/singleheader/simdjson.cpp)
# target_link_libraries(singleheader simdjson simdjson-flags)
# add_test(singleheader singleheader)
if(NOT MSVC)
######
# This tests is to guard us against ever again removing the soversion
# number from the library. See Bug
# https://github.com/simdjson/simdjson/issues/1014
#####
get_target_property(REPORTED_SIMDJSON_SOVERSION simdjson SOVERSION)
if(NOT ${REPORTED_SIMDJSON_SOVERSION} STREQUAL ${SIMDJSON_LIB_SOVERSION})
message(FATAL_ERROR "The library target does not have the proper soversion information." )
endif()
get_target_property(REPORTED_SIMDJSON_VERSION simdjson VERSION)
if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION})
message(FATAL_ERROR "The library target does not have the proper version information." )
endif()
endif()
# This tests validates that the implementation is what we think it is if we get passed
# SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing
add_cpp_test(checkimplementation LABELS other per_implementation)
add_subdirectory(compilation_failure_tests)
tests/dom/CMakeLists.txt
+155
View File
@@ -0,0 +1,155 @@
#
# These tests explicitly do #include "simdjson.cpp" so they can override stuff
#
if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL)
add_cpp_test(numberparsingcheck LABELS acceptance per_implementation)
simdjson_apply_props(numberparsingcheck)
target_link_libraries(numberparsingcheck PRIVATE simdjson-windows-headers)
target_compile_definitions(numberparsingcheck PRIVATE NOMINMAX)
target_include_directories(
numberparsingcheck
PRIVATE "${PROJECT_SOURCE_DIR}/src"
)
add_cpp_test(stringparsingcheck LABELS acceptance per_implementation)
simdjson_apply_props(numberparsingcheck)
target_link_libraries(stringparsingcheck PRIVATE simdjson-windows-headers)
target_compile_definitions(stringparsingcheck PRIVATE NOMINMAX)
target_include_directories(
stringparsingcheck
PRIVATE "${PROJECT_SOURCE_DIR}/src"
)
endif()
link_libraries(simdjson)
include_directories(..)
if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL)
add_cpp_test(random_string_number_tests LABELS dom acceptance per_implementation)
endif()
add_cpp_test(basictests LABELS dom acceptance per_implementation)
add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation)
add_cpp_test(document_tests LABELS dom acceptance per_implementation)
add_cpp_test(errortests LABELS dom acceptance per_implementation)
add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation)
add_cpp_test(integer_tests LABELS dom acceptance per_implementation)
add_cpp_test(jsoncheck LABELS dom acceptance per_implementation)
add_cpp_test(minefieldcheck LABELS dom acceptance per_implementation)
add_cpp_test(parse_many_test LABELS dom acceptance per_implementation)
add_cpp_test(pointercheck LABELS dom acceptance per_implementation) # https://tools.ietf.org/html/rfc6901
add_cpp_test(trivially_copyable_test LABELS dom acceptance per_implementation)
find_program(BASH bash)
# Below we skip anything on Windows, not just visual studio, because running bash under Windows requires you to
# map app paths to their "mounted" equivalent (e.g., /mnt/c/...). So even if you have bash under Windows, extra work would be
# required to make things work robustly. Simply put: bash is not quite portable.
# Script tests
if (BASH AND (NOT WIN32) AND SIMDJSON_BASH AND (TARGET json2json)) # The scripts are not robust enough to run under Windows even if bash is available
#
# json2json test
#
add_test(
NAME testjson2json
COMMAND ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/testjson2json.sh
WORKING_DIRECTORY $<TARGET_FILE_DIR:minify>
)
set_property(TEST testjson2json APPEND PROPERTY DEPENDS minify json2json)
set_property(TEST testjson2json APPEND PROPERTY LABELS per_implementation)
add_dependencies(per_implementation_tests minify)
#
# Competition parse test
#
if ((SIMDJSON_COMPETITION) AND (!SIMDJSON_SANITIZE))
# It looks like RapidJSON does not pass the sanitizer under some conditions (Clang 10)
add_executable(allparserscheckfile allparserscheckfile.cpp)
add_dependencies(competition_tests allparserscheckfile)
add_dependencies(per_implementation_tests allparserscheckfile)
target_link_libraries(allparserscheckfile PRIVATE competition-all)
add_test(issue150 ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/issue150.sh)
set_property(TEST issue150 APPEND PROPERTY LABELS per_implementation competition)
add_dependencies(all_tests allparserscheckfile)
endif()
#
# json2json tool test: check that json2json can parse twitter.json
#
add_test(NAME json2json COMMAND $<TARGET_FILE:json2json> ${EXAMPLE_JSON})
set_property(TEST json2json APPEND PROPERTY LABELS acceptance per_implementation)
add_dependencies(acceptance_tests json2json)
add_dependencies(per_implementation_tests json2json)
#
# SIMDJSON_FORCE_IMPLEMENTATION tests: run json2json with SIMDJSON
#
if (SIMDJSON_IMPLEMENTATION MATCHES "fallback")
add_test(
NAME simdjson_force_implementation
COMMAND
${CMAKE_COMMAND} -E env
SIMDJSON_FORCE_IMPLEMENTATION=fallback
$<TARGET_FILE:checkimplementation>
)
endif()
add_test(
NAME simdjson_force_implementation_error
COMMAND
${CMAKE_COMMAND} -E env
SIMDJSON_FORCE_IMPLEMENTATION=doesnotexist
$<TARGET_FILE:json2json> ${EXAMPLE_JSON}
)
set_tests_properties(simdjson_force_implementation_error PROPERTIES WILL_FAIL TRUE)
endif()
#
# Compile-only tests with simdjson flags on
#
# Don't add the tests if we're on VS2017 or older; they don't succeed.
if(NOT (MSVC AND MSVC_VERSION LESS 1920))
if(SIMDJSON_EXCEPTIONS)
add_cpp_test(readme_examples COMPILE_ONLY LABELS acceptance)
add_cpp_test(readme_examples11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
set_target_properties(readme_examples11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
endif()
add_cpp_test(readme_examples_noexceptions COMPILE_ONLY LABELS acceptance)
add_cpp_test(readme_examples_noexceptions11 COMPILE_ONLY LABELS acceptance SOURCES readme_examples_noexceptions.cpp)
set_target_properties(readme_examples_noexceptions11 PROPERTIES CXX_STANDARD 11 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
# Compile tests that *should fail*
add_cpp_test(readme_examples_will_fail_with_exceptions_off WILL_FAIL COMPILE_ONLY LABELS acceptance SOURCES readme_examples.cpp)
target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0)
endif()
if(MSVC AND BUILD_SHARED_LIBS)
add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
"$<TARGET_FILE_DIR:basictests>") # <--this is out-file path
endif()
if(NOT MSVC)
######
# This tests is to guard us against ever again removing the soversion
# number from the library. See Bug
# https://github.com/simdjson/simdjson/issues/1014
#####
get_target_property(REPORTED_SIMDJSON_SOVERSION simdjson SOVERSION)
if(NOT ${REPORTED_SIMDJSON_SOVERSION} STREQUAL ${SIMDJSON_LIB_SOVERSION})
message(FATAL_ERROR "The library target does not have the proper soversion information." )
endif()
get_target_property(REPORTED_SIMDJSON_VERSION simdjson VERSION)
if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION})
message(FATAL_ERROR "The library target does not have the proper version information." )
endif()
endif()
tests/allparserscheckfile.cpp → tests/dom/allparserscheckfile.cpp
View File
tests/basictests.cpp → tests/dom/basictests.cpp
+2 -2
View File
@@ -687,7 +687,7 @@ namespace dom_api_tests {
return false;
}
if (!iter.down()) {
printf("Root should not be emtpy\n");
printf("Root should not be empty\n");
return false;
}
if (!iter.is_string()) {
@@ -708,7 +708,7 @@ namespace dom_api_tests {
return false;
}
if(!iter.down()) {
printf("Image key should not be emtpy\n");
printf("Image key should not be empty\n");
return false;
}
if(!iter.next()) {
tests/document_stream_tests.cpp → tests/dom/document_stream_tests.cpp
View File
tests/document_tests.cpp → tests/dom/document_tests.cpp
+1 -1
View File
@@ -36,7 +36,7 @@ namespace document_tests {
return true;
}
// adversarial example that once triggred overruns, see https://github.com/lemire/simdjson/issues/345
// adversarial example that once triggered overruns, see https://github.com/simdjson/simdjson/issues/345
bool bad_example() {
std::cout << __func__ << std::endl;
simdjson::padded_string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"_padded;
tests/errortests.cpp → tests/dom/errortests.cpp
View File
tests/extracting_values_example.cpp → tests/dom/extracting_values_example.cpp
View File
tests/integer_tests.cpp → tests/dom/integer_tests.cpp
View File
tests/issue150.sh → tests/dom/issue150.sh
View File
tests/jsoncheck.cpp → tests/dom/jsoncheck.cpp
View File
tests/minefieldcheck.cpp → tests/dom/minefieldcheck.cpp
View File
tests/numberparsingcheck.cpp → tests/dom/numberparsingcheck.cpp
View File
tests/parse_many_test.cpp → tests/dom/parse_many_test.cpp
+1 -1
View File
@@ -58,7 +58,7 @@ bool validate(const char *dirname) {
bool everything_fine = true;
const char *extension1 = ".ndjson";
const char *extension2 = ".jsonl";
const char *extension3 = ".json"; // bad json files shoud fail
const char *extension3 = ".json"; // bad json files should fail
size_t dirlen = std::strlen(dirname);
struct dirent **entry_list;
tests/pointercheck.cpp → tests/dom/pointercheck.cpp
View File
tests/random_string_number_tests.cpp → tests/dom/random_string_number_tests.cpp
View File
tests/readme_examples.cpp → tests/dom/readme_examples.cpp
View File
tests/readme_examples_noexceptions.cpp → tests/dom/readme_examples_noexceptions.cpp
View File
tests/singleheadertest.cpp → tests/dom/singleheadertest.cpp
View File
tests/stringparsingcheck.cpp → tests/dom/stringparsingcheck.cpp
View File
tests/testjson2json.sh → tests/dom/testjson2json.sh
View File
tests/trivially_copyable_test.cpp → tests/dom/trivially_copyable_test.cpp
View File
tests/ondemand/CMakeLists.txt
+2 -2
View File
@@ -26,9 +26,9 @@ if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which
endif()
# Copy the simdjson dll into the tests directory
if(MSVC)
if(MSVC AND BUILD_SHARED_LIBS)
add_custom_command(TARGET ondemand_parse_api_tests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
"$<TARGET_FILE_DIR:ondemand_parse_api_tests>") # <--this is out-file path
endif(MSVC)
endif(MSVC AND BUILD_SHARED_LIBS)
tests/ondemand/compilation_failure_tests/CMakeLists.txt
+1 -1
View File
@@ -8,7 +8,7 @@
# adds a compilation test. Two targets are created, one expected to
# succeed compilation and one that is expected to fail.
function(add_dual_compile_test TEST_NAME)
add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY LABELS onedemand no_mingw)
add_cpp_test(${TEST_NAME}_should_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY LABELS ondemand no_mingw)
add_cpp_test(${TEST_NAME}_should_not_compile SOURCES ${TEST_NAME}.cpp COMPILE_ONLY WILL_FAIL LABELS ondemand acceptance no_mingw)
target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1)
endfunction(add_dual_compile_test)
tools/CMakeLists.txt
+1 -1
View File
@@ -1,5 +1,5 @@
if(TARGET cxxopts) # we only build the tools if cxxopts is available
message(STATUS "We have cxxopts as a dependency and we are buiding the tools (e.g., json2json).")
message(STATUS "We have cxxopts as a dependency and we are building the tools (e.g., json2json).")
foreach(tool IN ITEMS json2json jsonstats jsonpointer minify)
add_executable("${tool}" "${tool}.cpp")
simdjson_apply_props("${tool}")
tools/release.py
+1 -1
View File
@@ -145,7 +145,7 @@ if(atleastminor):
sonumber += 1
for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
line = re.sub(' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line)
line = re.sub(' VERSION \d+\.\d+\.\d+',' VERSION '+newmajorversionstring+'.'+mewminorversionstring+'.'+newrevversionstring, line.rstrip())
line = re.sub('SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
line = re.sub('set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
print(line)