mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
Compare commits
46 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 84b01e6346 | |||
| 90e1027bc6 | |||
| 41b56b2f2a | |||
| e1eb5d9d30 | |||
| 3b8b7e89e4 | |||
| b54039bb25 | |||
| 75178b65e0 | |||
| 7453fbe878 | |||
| 9d767a9f59 | |||
| b015f8959b | |||
| 8d06129667 | |||
| a4f3a08c65 | |||
| 6fe66d0fa7 | |||
| 1aa1d537fe | |||
| c122462278 | |||
| b6b338bea9 | |||
| f72636b5b0 | |||
| f6a2bac11b | |||
| e4ddc5446b | |||
| e0e0aa9e7b | |||
| 9f0a2e0e86 | |||
| 203c03e90b | |||
| 6698eb96b9 | |||
| 30f7de387c | |||
| e0788507fe | |||
| e4740c87fb | |||
| 5beef701e7 | |||
| c6f9c93c33 | |||
| 3a93e45dd1 | |||
| 6db8ceb46c | |||
| 23651f0d41 | |||
| 7e1893db42 | |||
| b4b1927b0f | |||
| e275589832 | |||
| 69ee84973a | |||
| 76bed68ec3 | |||
| 003d970cd2 | |||
| 4aaf3f030c | |||
| 2763f7e719 | |||
| 39035c39be | |||
| ac179ff40f | |||
| b41bece32c | |||
| 9cb0c5f92d | |||
| 1dec7b04fe | |||
| c6eb6f98b8 | |||
| 1b01969bda |
+8
-4
@@ -8,7 +8,7 @@ steps:
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
BUILD_FLAGS: -- -j
|
||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
|
||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||
commands:
|
||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||
@@ -19,6 +19,7 @@ steps:
|
||||
- cmake $CMAKE_FLAGS ..
|
||||
- cmake --build . $BUILD_FLAGS
|
||||
- ctest $CTEST_FLAGS -L acceptance -LE per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
|
||||
@@ -35,7 +36,7 @@ steps:
|
||||
CC: clang-6.0
|
||||
CXX: clang++-6.0
|
||||
BUILD_FLAGS: -- -j
|
||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
|
||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||
commands:
|
||||
- mkdir build
|
||||
@@ -43,6 +44,7 @@ steps:
|
||||
- cmake $CMAKE_FLAGS ..
|
||||
- cmake --build . $BUILD_FLAGS
|
||||
- ctest $CTEST_FLAGS -L acceptance -LE per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
|
||||
@@ -100,7 +102,7 @@ steps:
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
BUILD_FLAGS: -- -j
|
||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
|
||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||
commands:
|
||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||
@@ -111,6 +113,7 @@ steps:
|
||||
- cmake $CMAKE_FLAGS ..
|
||||
- cmake --build . $BUILD_FLAGS
|
||||
- ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
@@ -126,7 +129,7 @@ steps:
|
||||
environment:
|
||||
CC: clang-9
|
||||
CXX: clang++-9
|
||||
CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||
CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=icelake;haswell;westmere;fallback
|
||||
BUILD_FLAGS: -- -j
|
||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||
commands:
|
||||
@@ -135,6 +138,7 @@ steps:
|
||||
- cmake $CMAKE_FLAGS ..
|
||||
- cmake --build . $BUILD_FLAGS
|
||||
- ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L acceptance -LE per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=icelake ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=haswell ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=westmere ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ASAN_OPTIONS="detect_leaks=0" ctest $CTEST_FLAGS -L per_implementation
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
name: CIFuzz
|
||||
on: [pull_request]
|
||||
jobs:
|
||||
Fuzzing:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Build Fuzzers
|
||||
id: build
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'simdjson'
|
||||
dry-run: false
|
||||
- name: Run Fuzzers
|
||||
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
|
||||
with:
|
||||
oss-fuzz-project-name: 'simdjson'
|
||||
fuzz-seconds: 600
|
||||
dry-run: false
|
||||
- name: Upload Crash
|
||||
uses: actions/upload-artifact@v1
|
||||
if: failure() && steps.build.outcome == 'success'
|
||||
with:
|
||||
name: artifacts
|
||||
path: ./out/artifacts
|
||||
@@ -1,63 +0,0 @@
|
||||
name: MinGW32-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
|
||||
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
|
||||
|
||||
# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
|
||||
# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
name: windows-gcc
|
||||
runs-on: windows-2016
|
||||
|
||||
env:
|
||||
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
|
||||
steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/cache@v2 # we cache the scoop setup with 32-bit GCC
|
||||
id: cache
|
||||
with:
|
||||
path: |
|
||||
C:\ProgramData\scoop
|
||||
key: scoop32 # static key: should be good forever
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
|
||||
- name: Setup Windows # This should almost never run if the cache works.
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
shell: powershell
|
||||
run: |
|
||||
Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
|
||||
scoop install sudo --global
|
||||
sudo scoop install git --global
|
||||
sudo scoop install ninja --global
|
||||
sudo scoop install cmake --global
|
||||
sudo scoop install gcc --arch 32bit --global
|
||||
$env:path
|
||||
Write-Host 'Everything has been installed, you are good!'
|
||||
- name: Build and Test 32-bit x86
|
||||
shell: powershell
|
||||
run: |
|
||||
$ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
|
||||
g++ --version
|
||||
cmake --version
|
||||
ninja --version
|
||||
git --version
|
||||
mkdir build32
|
||||
cd build32
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
|
||||
cmake --build . --target acceptance_tests --verbose
|
||||
ctest -L acceptance --output-on-failure
|
||||
@@ -1,69 +0,0 @@
|
||||
name: MinGW64-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
|
||||
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
|
||||
|
||||
# It is important to build static libraries because cmake is not smart enough under Windows/mingw to take care of the path. So
|
||||
# with a dynamic library, you could get failures due to the fact that the EXE can't find its DLL.
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
name: windows-gcc
|
||||
runs-on: windows-2016
|
||||
|
||||
env:
|
||||
CMAKE_GENERATOR: Ninja # This is critical, try ' cmake -GNinja-DBUILD_SHARED_LIBS=OFF .. ' if using the command line
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
|
||||
steps: # To reproduce what is below, start a powershell with administrative rights, using scoop *is* a good idea
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- uses: actions/cache@v2 # we cache the scoop setup with 64-bit GCC
|
||||
id: cache
|
||||
with:
|
||||
path: |
|
||||
C:\ProgramData\scoop
|
||||
key: scoop64 # static key: should be good forever
|
||||
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
|
||||
- name: Setup Windows # This should almost never run if the cache works.
|
||||
if: steps.cache.outputs.cache-hit != 'true'
|
||||
shell: powershell
|
||||
run: |
|
||||
Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
|
||||
scoop install sudo --global
|
||||
sudo scoop install git --global
|
||||
sudo scoop install ninja --global
|
||||
sudo scoop install cmake --global
|
||||
sudo scoop install gcc --arch 64bit --global
|
||||
$env:path
|
||||
Write-Host 'Everything has been installed, you are good!'
|
||||
- name: Build and Test 64-bit x64
|
||||
shell: powershell
|
||||
run: |
|
||||
$ENV:PATH="C:\ProgramData\scoop\shims;C:\ProgramData\scoop\apps\gcc\current\bin;C:\ProgramData\scoop\apps\ninja\current;$ENV:PATH"
|
||||
g++ --version
|
||||
cmake --version
|
||||
ninja --version
|
||||
git --version
|
||||
mkdir build64
|
||||
cd build64
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
|
||||
cmake --build . --target acceptance_tests --verbose
|
||||
ctest -L acceptance --output-on-failure
|
||||
cd ..
|
||||
mkdir build64debug
|
||||
cd build64debug
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_ENABLE_THREADS=OFF ..
|
||||
cmake --build . --target acceptance_tests --verbose
|
||||
ctest -L acceptance --output-on-failure
|
||||
@@ -38,6 +38,6 @@ jobs:
|
||||
run: |
|
||||
mkdir build
|
||||
cd build
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DSIMDJSON_BUILD_STATIC=ON -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
|
||||
cmake -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DO_NOT_USE_THREADS_NO_MATTER_WHAT=ON ..
|
||||
cmake --build . --verbose
|
||||
ctest -j4 --output-on-failure -LE explicitonly
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Ubuntu 22.04 CI (GCC 11)
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
ubuntu-build:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
- name: Use cmake
|
||||
run: |
|
||||
mkdir builddebug &&
|
||||
cd builddebug &&
|
||||
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
cd .. &&
|
||||
mkdir build &&
|
||||
cd build &&
|
||||
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
|
||||
cmake --build . &&
|
||||
ctest -j --output-on-failure -LE explicitonly &&
|
||||
cmake --install . &&
|
||||
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
|
||||
cd ../tests/installation_tests/find &&
|
||||
mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build .
|
||||
@@ -1,31 +0,0 @@
|
||||
name: VS16-CLANG-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
name: windows-vs16
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
- name: 'Run CMake with VS16 Clang'
|
||||
uses: lukka/run-cmake@v3
|
||||
with:
|
||||
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
|
||||
cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
|
||||
buildDirectory: "${{ github.workspace }}/../../_temp/windows"
|
||||
cmakeBuildType: Release
|
||||
buildWithCMake: true
|
||||
cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
|
||||
buildWithCMakeArgs: --config Release
|
||||
|
||||
- name: 'Run CTest'
|
||||
run: ctest -C Release -LE explicitonly --output-on-failure
|
||||
working-directory: "${{ github.workspace }}/../../_temp/windows"
|
||||
@@ -1,45 +0,0 @@
|
||||
name: VS16-Ninja-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
name: windows-vs16
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: actions/cache@v2
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
- name: 'Run CMake with VS16'
|
||||
uses: lukka/run-cmake@v2
|
||||
with:
|
||||
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
|
||||
cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
|
||||
buildDirectory: "${{ github.workspace }}/../../_temp/windows"
|
||||
cmakeBuildType: Release
|
||||
buildWithCMake: true
|
||||
cmakeAppendedArgs: -G Ninja -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
|
||||
buildWithCMakeArgs: --config Release
|
||||
- name: 'Run CTest'
|
||||
run: ctest -C Release -LE explicitonly --output-on-failure
|
||||
working-directory: "${{ github.workspace }}/../../_temp/windows"
|
||||
- name: 'Install with CMake'
|
||||
uses: lukka/run-cmake@v3
|
||||
with:
|
||||
cmakeListsTxtPath: '${{ github.workspace }}/CMakeLists.txt'
|
||||
buildWithCMakeArgs: '--target install'
|
||||
- name: 'Test Installation with CMake'
|
||||
uses: lukka/run-cmake@v3
|
||||
with:
|
||||
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
|
||||
cmakeListsTxtPath: '${{ github.workspace }}/tests/installation_tests/find/CMakeLists.txt'
|
||||
cmakeBuildType: Release
|
||||
buildWithCMake: true
|
||||
buildDirectory: '${{ github.workspace }}/tests/installation_tests/find/buildDirectory'
|
||||
cmakeAppendedArgs: -G Ninja
|
||||
buildWithCMakeArgs: '--config Release --verbose'
|
||||
@@ -1,10 +1,10 @@
|
||||
name: VS16-ARM-CI
|
||||
name: VS17-ARM-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
name: windows-vs16
|
||||
name: windows-vs17
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -1,4 +1,4 @@
|
||||
name: VS16-CI
|
||||
name: VS17-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
@@ -7,22 +7,22 @@ jobs:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
name: windows-vs16
|
||||
name: windows-vs17
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- {gen: Visual Studio 16 2019, arch: Win32, static: ON}
|
||||
- {gen: Visual Studio 16 2019, arch: Win32, static: OFF}
|
||||
- {gen: Visual Studio 16 2019, arch: x64, static: ON}
|
||||
- {gen: Visual Studio 16 2019, arch: x64, static: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: Win32, shared: ON}
|
||||
- {gen: Visual Studio 17 2022, arch: Win32, shared: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: ON}
|
||||
- {gen: Visual Studio 17 2022, arch: x64, shared: OFF}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Configure
|
||||
run: |
|
||||
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
|
||||
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
|
||||
- name: Build Debug
|
||||
run: cmake --build build --config Debug --verbose
|
||||
- name: Build Release
|
||||
@@ -1,4 +1,4 @@
|
||||
name: VS15-CI
|
||||
name: VS17-CLANG-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
@@ -7,22 +7,19 @@ jobs:
|
||||
if: >-
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||
name: windows-vs15
|
||||
runs-on: windows-2016
|
||||
name: windows-vs17
|
||||
runs-on: windows-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- {gen: Visual Studio 15 2017, arch: Win32, static: ON}
|
||||
- {gen: Visual Studio 15 2017, arch: Win32, static: OFF}
|
||||
- {gen: Visual Studio 15 2017, arch: x64, static: ON}
|
||||
- {gen: Visual Studio 15 2017, arch: x64, static: OFF}
|
||||
- {gen: Visual Studio 17 2022, arch: x64}
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Configure
|
||||
run: |
|
||||
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_BUILD_STATIC=${{matrix.static}} -B build
|
||||
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
|
||||
- name: Build Debug
|
||||
run: cmake --build build --config Debug --verbose
|
||||
- name: Build Release
|
||||
@@ -1,10 +1,10 @@
|
||||
name: VS16-NoExcept-CI
|
||||
name: VS17-NoExcept-CI
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
name: windows-vs16
|
||||
name: windows-vs17
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -12,7 +12,7 @@ jobs:
|
||||
with:
|
||||
path: dependencies/.cache
|
||||
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||
- name: 'Run CMake with VS16'
|
||||
- name: 'Run CMake with VS17'
|
||||
uses: lukka/run-cmake@v3
|
||||
with:
|
||||
cmakeListsOrSettingsJson: CMakeListsTxtAdvanced
|
||||
+2
-10
@@ -49,19 +49,10 @@ endif()
|
||||
if(is_top_project)
|
||||
option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF)
|
||||
option(BUILD_SHARED_LIBS "Build simdjson as a shared library" OFF)
|
||||
|
||||
if("$ENV{CI}")
|
||||
set(SIMDJSON_DEVELOPER_MODE ON CACHE INTERNAL "")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
include(cmake/handle-deprecations.cmake)
|
||||
|
||||
if(SIMDJSON_DEVELOPER_MODE)
|
||||
include(cmake/developer-options.cmake)
|
||||
else()
|
||||
message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
|
||||
endif()
|
||||
include(cmake/developer-options.cmake)
|
||||
|
||||
# ---- simdjson library ----
|
||||
|
||||
@@ -180,6 +171,7 @@ endif()
|
||||
# ---- Developer mode extras ----
|
||||
|
||||
if(NOT SIMDJSON_DEVELOPER_MODE)
|
||||
message(STATUS "Building only the library. Advanced users may want to turn SIMDJSON_DEVELOPER_MODE to ON, e.g., via -D SIMDJSON_DEVELOPER_MODE=ON.")
|
||||
return()
|
||||
elseif(NOT is_top_project)
|
||||
message(AUTHOR_WARNING "Developer mode is intended for developers of simdjson")
|
||||
|
||||
+5
-1
@@ -37,5 +37,9 @@ Furkan Taşkale
|
||||
Brendan Knapp
|
||||
Danila Kutenin
|
||||
Pavel Pavlov
|
||||
# if you have contributed to the project and your name does not
|
||||
Hao Chen
|
||||
Nicolas Boyer
|
||||
Kim Walisch and Jatin Bhateja (AVX-512 bitset decoder)
|
||||
Eric Zhang (AVX-512 kernel)
|
||||
# if you have contributed to the project and your name does not
|
||||
# appear in this list, please let us know!
|
||||
|
||||
@@ -138,6 +138,7 @@ We distinguish between "bindings" (which just wrap the C++ code) and a port to a
|
||||
- [rcppsimdjson](https://github.com/eddelbuettel/rcppsimdjson): R bindings.
|
||||
- [simdjson_erlang](https://github.com/ChomperT/simdjson_erlang): erlang bindings.
|
||||
- [lua-simdjson](https://github.com/FourierTransformer/lua-simdjson): lua bindings.
|
||||
- [hermes-json](https://hackage.haskell.org/package/hermes-json): haskell bindings.
|
||||
|
||||
|
||||
About simdjson
|
||||
|
||||
@@ -97,9 +97,9 @@ struct option_struct {
|
||||
verbose = true;
|
||||
break;
|
||||
case 'a': {
|
||||
auto impl = simdjson::available_implementations[optarg];
|
||||
auto impl = simdjson::get_available_implementations()[optarg];
|
||||
if(impl && impl->supported_by_runtime_system()) {
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
} else {
|
||||
std::cerr << "implementation " << optarg << " not found or not supported " << std::endl;
|
||||
}
|
||||
|
||||
@@ -66,7 +66,7 @@ void print_usage(ostream& out) {
|
||||
out << "-H - Make the buffers hot (reduce page allocation and related OS tasks during parsing) [default]" << endl;
|
||||
out << "-a IMPL - Use the given parser implementation. By default, detects the most advanced" << endl;
|
||||
out << " implementation supported on the host machine." << endl;
|
||||
for (auto impl : simdjson::available_implementations) {
|
||||
for (auto impl : simdjson::get_available_implementations()) {
|
||||
if(impl->supported_by_runtime_system()) {
|
||||
out << "-a " << std::left << std::setw(9) << impl->name() << " - Use the " << impl->description() << " parser implementation." << endl;
|
||||
}
|
||||
@@ -116,10 +116,10 @@ struct option_struct {
|
||||
verbose = true;
|
||||
break;
|
||||
case 'a': {
|
||||
const implementation *impl = simdjson::available_implementations[optarg];
|
||||
const implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if ((!impl) || (!impl->supported_by_runtime_system())) {
|
||||
std::string exit_message = string("Unsupported option value -a ") + optarg + ": expected -a with one of ";
|
||||
for (auto imple : simdjson::available_implementations) {
|
||||
for (auto imple : simdjson::get_available_implementations()) {
|
||||
if(imple->supported_by_runtime_system()) {
|
||||
exit_message += imple->name();
|
||||
exit_message += " ";
|
||||
@@ -127,7 +127,7 @@ struct option_struct {
|
||||
}
|
||||
exit_usage(exit_message);
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
case 'C':
|
||||
@@ -175,7 +175,7 @@ int main(int argc, char *argv[]) {
|
||||
option_struct options(argc, argv);
|
||||
if (options.verbose) {
|
||||
verbose_stream = &cout;
|
||||
verbose() << "Implementation: " << simdjson::active_implementation->name() << endl;
|
||||
verbose() << "Implementation: " << simdjson::get_active_implementation()->name() << endl;
|
||||
}
|
||||
|
||||
// Start collecting events. We put this early so if it prints an error message, it's the
|
||||
|
||||
@@ -50,9 +50,9 @@ simdjson_really_inline void simdjson_process_atom(stat_t &s,
|
||||
} else if(element.is<double>()) {
|
||||
s.float_count++;
|
||||
} else if (element.is<bool>()) {
|
||||
simdjson::error_code err;
|
||||
bool v;
|
||||
err = element.get(v);
|
||||
simdjson::error_code error;
|
||||
if ((error = element.get(v))) { std::cerr << error << std::endl; abort(); }
|
||||
if (v) {
|
||||
s.true_count++;
|
||||
} else {
|
||||
|
||||
@@ -10,8 +10,8 @@ void maybe_display_implementation() {
|
||||
static bool displayed_implementation = false;
|
||||
if(!displayed_implementation) {
|
||||
displayed_implementation = true;
|
||||
std::cout << "simdjson::dom implementation: " << simdjson::active_implementation->name() << std::endl;
|
||||
std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::active_implementation->name() << std::endl;
|
||||
std::cout << "simdjson::dom implementation: " << simdjson::get_active_implementation()->name() << std::endl;
|
||||
std::cout << "simdjson::ondemand implementation (stage 1): " << simdjson::get_active_implementation()->name() << std::endl;
|
||||
std::cout << "simdjson::ondemand implementation (stage 2): " << simdjson::builtin_implementation()->name() << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,6 +175,18 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel")
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-intel")
|
||||
endif()
|
||||
|
||||
option(
|
||||
SIMDJSON_AVX512_ALLOWED
|
||||
"Enable AVX-512 instructions (only affects processors and compilers with AVX-512 support)."
|
||||
ON
|
||||
)
|
||||
if(SIMDJSON_AVX512_ALLOWED)
|
||||
add_compile_definitions(SIMDJSON_AVX512_ALLOWED=1)
|
||||
else()
|
||||
add_compile_definitions(SIMDJSON_AVX512_ALLOWED=0)
|
||||
message(STATUS "AVX-512 instructions are not allowed.")
|
||||
endif()
|
||||
|
||||
include(CheckSymbolExists)
|
||||
check_symbol_exists(fork unistd.h HAVE_POSIX_FORK)
|
||||
check_symbol_exists(wait sys/wait.h HAVE_POSIX_WAIT)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#
|
||||
# Implementation selection
|
||||
#
|
||||
set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell arm64 ppc64)
|
||||
set(SIMDJSON_ALL_IMPLEMENTATIONS fallback westmere haswell icelake arm64 ppc64)
|
||||
|
||||
set(
|
||||
SIMDJSON_IMPLEMENTATION ""
|
||||
@@ -15,7 +15,7 @@ set(
|
||||
SIMDJSON_EXCLUDE_IMPLEMENTATION ""
|
||||
CACHE STRING "\
|
||||
Semicolon-separated list of implementations to exclude \
|
||||
(haswell/westmere/arm64/ppc64/fallback). By default, excludes any \
|
||||
(icelake/haswell/westmere/arm64/ppc64/fallback). By default, excludes any \
|
||||
implementations that are unsupported at compile time or cannot be selected at \
|
||||
runtime."
|
||||
)
|
||||
|
||||
+40
-22
@@ -16,10 +16,10 @@ An overview of what you need to know to use simdjson, with examples.
|
||||
* [UTF-8 validation (alone)](#utf-8-validation-alone)
|
||||
* [JSON Pointer](#json-pointer)
|
||||
* [Error Handling](#error-handling)
|
||||
* [Error Handling Example without Exceptions](#error-handling-example-without-exceptions)
|
||||
* [Error Handling Example without Exceptions](#error-handling-examples-without-exceptions)
|
||||
* [Disabling Exceptions](#disabling-exceptions)
|
||||
* [Exceptions](#exceptions)
|
||||
* [Current location in document](#current-location-in-documnet)
|
||||
* [Current location in document](#current-location-in-document)
|
||||
* [Rewinding](#rewinding)
|
||||
* [Direct Access to the Raw String](#direct-access-to-the-raw-string)
|
||||
* [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines)
|
||||
@@ -54,6 +54,7 @@ c++ myproject.cpp simdjson.cpp
|
||||
|
||||
Note:
|
||||
- Users on macOS and other platforms where default compilers do not provide C++11 compliant by default should request it with the appropriate flag (e.g., `c++ -std=c++17 myproject.cpp simdjson.cpp`).
|
||||
- The library relies on [runtime CPU detection](implementation-selection.md): avoid specifying an architecture at compile time (e.g., `-march-native`).
|
||||
|
||||
Using simdjson with package managers
|
||||
------------------
|
||||
@@ -200,7 +201,8 @@ allocations during parsing when using simdjson. [See our performance notes for d
|
||||
C++11 Support and string_view
|
||||
-------------
|
||||
|
||||
The simdjson library builds on compilers supporting the [C++11 standard](https://en.wikipedia.org/wiki/C%2B%2B11). It is also a strict requirement: we have no plan to support older C++ compilers.
|
||||
The simdjson library builds on compilers supporting the [C++11 standard](https://en.wikipedia.org/wiki/C%2B%2B11).
|
||||
It is also a strict requirement: we have no plan to support older C++ compilers.
|
||||
|
||||
We represent parsed Unicode (UTF-8) strings in simdjson using the `std::string_view` class. It avoids
|
||||
the need to copy the data, as would be necessary with the `std::string` class. It also
|
||||
@@ -218,7 +220,6 @@ is often best viewed as a temporary string value that is tied to the document yo
|
||||
At the cost of some memory allocation, you may convert your `std::string_view` instances for long-term storage into `std::string` instances:
|
||||
`std::string mycopy(view)` (C++17) or `std::string mycopy(view.begin(), view.end())` (prior to C++17).
|
||||
|
||||
|
||||
The `std::string_view` class has become standard as part of C++17 but it is not always available
|
||||
on compilers which only supports C++11. When we detect that `string_view` is natively
|
||||
available, we define the macro `SIMDJSON_HAS_STRING_VIEW`.
|
||||
@@ -236,6 +237,8 @@ transcode the UTF-8 strings produced by the simdjson library to other formats. S
|
||||
Using the Parsed JSON
|
||||
---------------------
|
||||
|
||||
|
||||
|
||||
We recommend that you first compile and run your code in Debug mode (with `NDEBUG`
|
||||
undefined). When you do so, the simdjson library runs additional sanity tests on
|
||||
your code to help ensure that you are using the library in a safe manner. Once
|
||||
@@ -251,8 +254,10 @@ native types (`double`, `uint64_t`, `int64_t`, `bool`), we also access
|
||||
Unicode (UTF-8) strings (`std::string_view`), objects (`simdjson::ondemand::object`)
|
||||
and arrays (`simdjson::ondemand::array`).
|
||||
We also have a generic type (`simdjson::ondemand::value`) which represents a potential
|
||||
array or object, or scalar type (`double`, `uint64_t`, `int64_t`, `bool`, `null`, string) inside an array or an object. Both generic types (`simdjson::ondemand::document` and `simdjson::ondemand::value`) have a `type()` method returning
|
||||
a `json_type` value describing the value (`json_type::array`, `json_type::object`, `json_type::number`, `json_type::string`, `json_type::boolean`, `json_type::null`).
|
||||
array or object, or scalar type (`double`, `uint64_t`, `int64_t`, `bool`, `null`, string) inside
|
||||
an array or an object. Both generic types (`simdjson::ondemand::document` and
|
||||
`simdjson::ondemand::value`) have a `type()` method returning a `json_type` value describing the
|
||||
value (`json_type::array`, `json_type::object`, `json_type::number`, `json_type::string`, `json_type::boolean`, `json_type::null`).
|
||||
|
||||
Advanced users who need to determine the number types (integer or float) dynamically,
|
||||
should review our section [dynamic number types](#dynamic-number-types). Indeed,
|
||||
@@ -260,10 +265,10 @@ we have an additional `ondemand::number` type which may represent either integer
|
||||
or floating-point values, depending on how the numbers are formatted.
|
||||
floating-point values followed by an integer.
|
||||
|
||||
While you are accessing the document, the `document` instance should remain in scope:
|
||||
it is your "iterator" which keeps track of where you are in the JSON document.
|
||||
By design, there is one and only one `document` instance per JSON document.
|
||||
|
||||
We invite you to keep the following rules in mind:
|
||||
1. While you are accessing the document, the `document` instance should remain in scope: it is your "iterator" which keeps track of where you are in the JSON document. By design, there is one and only one `document` instance per JSON document.
|
||||
2. Because On Demand is really just an iterator, you must fully consume the current object or array before accessing a sibling object or array.
|
||||
3. Values can only be consumed once, you should get the values and store them if you plan to need them multiple times. You are expected to access the keys of an object just once. You are expected to go through the values of an array just once.
|
||||
|
||||
The following specific instructions indicate how to use the JSON when exceptions are enabled, but simdjson has full, idiomatic
|
||||
support for users who avoid exceptions. See [the simdjson error handling documentation](basics.md#error-handling) for more.
|
||||
@@ -273,18 +278,24 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
||||
However, it is not fully validated. On Demand only fully validates the values you use and the
|
||||
structure leading to it.
|
||||
* **Extracting Values:** You can cast a JSON element to a native type:
|
||||
`double(element)` or `double x = json_element`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
|
||||
ondemand::object and ondemand::array. At this point, the number, string or boolean will be parsed,
|
||||
or the initial `[` or `{` will be verified. An exception is thrown if the cast is not possible.
|
||||
`double(element)`. This works for `std::string_view`, double, uint64_t, int64_t, bool,
|
||||
ondemand::object and ondemand::array. We also have explicit methods such as `get_string()`, `get_double()`,
|
||||
`get_uint64()`, `get_int64()`, `get_bool()`, `get_object()` and `get_array()`. After a cast or an explicit method,
|
||||
the number, string or boolean will be parsed, or the initial `[` or `{` will be verified. An exception is thrown if
|
||||
the cast is not possible.
|
||||
|
||||
> IMPORTANT NOTE: values can only be parsed once. Since documents are *iterators*, once you have
|
||||
> parsed a value (such as by casting to double), you cannot get at it again.
|
||||
> parsed a value (such as by casting to double), you cannot get at it again. It is an error to call
|
||||
> `get_string()` twice on an object (or to cast an object twice to `std::string_view`).
|
||||
* **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`. This will
|
||||
scan through the object looking for the field with the matching string, doing a character-by-character
|
||||
comparison. For efficiency reason, you should avoid looking up the same field repeatedly: e.g., do
|
||||
not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. Furthermore, you can only consume one field at a time, on the same object. Thus
|
||||
if you have retrieved `content["bids"].get_array()` and you later call `content["asks"].get_array()`, then the first array should no longer be accessed: it would be unsafe to do so.
|
||||
You can detect such mistakes by first compiling and running the code in Debug mode: an OUT_OF_ORDER_ITERATION error is generated.
|
||||
not do `object["foo"]` followed by `object["foo"]` with the same `object` instance. If you consume an
|
||||
object twice: `std::string_view(object["foo"]` followed by `std::string_view(object["foo"]`, your code
|
||||
is in error. Furthermore, you can only consume one field at a time, on the same object. Thus
|
||||
if you have retrieved `content["bids"].get_array()` and you later call `content["asks"].get_array()`, then the
|
||||
first array should no longer be accessed: it would be unsafe to do so. You can detect such mistakes by first
|
||||
compiling and running the code in Debug mode: an OUT_OF_ORDER_ITERATION error is generated.
|
||||
|
||||
> NOTE: JSON allows you to escape characters in keys. E.g., the key `"date"` may be written as
|
||||
> `"\u0064\u0061\u0074\u0065"`. By default, simdjson does *not* unescape keys when matching by default.
|
||||
@@ -750,12 +761,16 @@ The entire simdjson API is usable with and without exceptions. All simdjson APIs
|
||||
pair. You can retrieve the value with .get() without generating an exception, like so:
|
||||
|
||||
```c++
|
||||
ondemand::element doc;
|
||||
ondemand::document doc;
|
||||
auto error = parser.iterate(json).get(doc);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
```
|
||||
|
||||
When you use the code this way, it is your responsibility to check for error before using the
|
||||
When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
|
||||
We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
|
||||
error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
|
||||
|
||||
When you use the code without exceptions, it is your responsibility to check for error before using the
|
||||
result: if there is an error, the result value will not be valid and using it will caused undefined behavior. Most compilers should be able to help you if you activate the right
|
||||
set of warnings: they can identify variables that are written to but never otherwise accessed.
|
||||
|
||||
@@ -877,6 +892,9 @@ int main(void) {
|
||||
}
|
||||
```
|
||||
|
||||
The `at` method can only be called once on an array. It cannot be used
|
||||
to iterate through the values of an array.
|
||||
|
||||
### Error Handling Examples without Exceptions
|
||||
|
||||
This is how the example in "Using the Parsed JSON" could be written using only error code checking (without exceptions):
|
||||
@@ -1128,7 +1146,7 @@ Direct Access to the Raw String
|
||||
|
||||
The simdjson library makes explicit assumptions about types. For examples, numbers
|
||||
must be integers (up to 64-bit integers) or binary64 floating-point numbers. Some users
|
||||
have different needs. For example, some users might want to support big infloating-point number followed by an integer.tegers.
|
||||
have different needs. For example, some users might want to support big integers.
|
||||
The library makes this possible by providing a `raw_json_token` method which returns
|
||||
a `std::string_view` instance containing the value as a string which you may then
|
||||
parse as you see fit.
|
||||
@@ -1455,8 +1473,8 @@ The simdjson library is fully compliant with the [RFC 8259](https://www.tbray.o
|
||||
- A single string or a single number is considered to be a valid JSON document.
|
||||
- We fully validate the numbers according to the JSON specification. For example, the string `01` is not valid JSON document since the specification states that *leading zeros are not allowed*.
|
||||
- The specification allows implementations to set limits on the range and precision of numbers accepted. We support 64-bit floating-point numbers as well as integer values.
|
||||
- We parse integers and floating-point numbers afloating-point number followed by an integer.s separate types which allows us to support all signed (two's complement) 64-bit integersfloating-point number followed by an integer., like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document.
|
||||
- We support the full range of 64-bit floating-point numbers (binary64). The values range from `std::numeric_limits<double>::lowest()` to `std::numefloating-point number followed by an integer.ric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document. Numbers are parsed with a perfect accuracy (ULP 0): the nearest floating-point value is chosen, rounding to even when needed. If you serialized your floating-point numbers with 17floating-point value followed by an integer. significant digits in a standard compliant manner, the simdjson library is guaranfloating-point number followed by an integer.teed to recover the same numbers, exactly.
|
||||
- We parse integers and floating-point numbers as separate types which allows us to support all signed (two's complement) 64-bit integers, like a Java `long` or a C/C++ `long long` and all 64-bit unsigned integers. When we cannot represent exactly an integer as a signed or unsigned 64-bit value, we reject the JSON document.
|
||||
- We support the full range of 64-bit floating-point numbers (binary64). The values range from `std::numeric_limits<double>::lowest()` to `std::numeric_limits<double>::max()`, so from -1.7976e308 all the way to 1.7975e308. Extreme values (less or equal to -1e308, greater or equal to 1e308) are rejected: we refuse to parse the input document. Numbers are parsed with a perfect accuracy (ULP 0): the nearest floating-point value is chosen, rounding to even when needed. If you serialized your floating-point numbers with 17 significant digits in a standard compliant manner, the simdjson library is guaranteed to recover the same numbers, exactly.
|
||||
- The specification states that JSON text exchanged between systems that are not part of a closed ecosystem MUST be encoded using UTF-8. The simdjson library does full UTF-8 validation as part of the parsing. The specification states that implementations MUST NOT add a byte order mark: the simdjson library rejects documents starting with a byte order mark.
|
||||
- The simdjson library validates string content for unescaped characters. Unescaped line breaks and tabs in strings are not allowed.
|
||||
- The simdjson library accepts objects with repeated keys: all of the name/value pairs, including duplicates, are reported. We do not enforce key uniqueness.
|
||||
|
||||
+11
-4
@@ -3,8 +3,8 @@ The Document-Object-Model (DOM) front-end
|
||||
|
||||
An overview of what you need to know to use simdjson, with examples.
|
||||
|
||||
* [DOM vs On Demand](#dom-vs-ondemand)
|
||||
* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents)
|
||||
* [DOM vs On Demand](#dom-vs-on-demand)
|
||||
* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents-using-the-dom-front-end)
|
||||
* [Using the Parsed JSON](#using-the-parsed-json)
|
||||
* [C++17 Support](#c17-support)
|
||||
* [JSON Pointer](#json-pointer)
|
||||
@@ -81,7 +81,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
|
||||
know the type of the value, you can cast it right there, too! `for (double value : array) { ... }`
|
||||
* **Object Iteration:** You can iterate through an object's fields, too: `for (auto [key, value] : object)`
|
||||
* **Array Index:** To get at an array value by index, use the at() method: `array.at(0)` gets the
|
||||
first element.
|
||||
first element. The at() method has linear-time complexity so it should not be used to iterate over the values of an array.
|
||||
> Note that array[0] does not compile, because implementing [] gives the impression indexing is a
|
||||
> O(1) operation, which it is not presently in simdjson. Instead, you should iterate over the elements
|
||||
> using a for-loop, as in our examples.
|
||||
@@ -249,7 +249,11 @@ auto error = parser.parse(json).get(doc);
|
||||
if (error) { cerr << error << endl; exit(1); }
|
||||
```
|
||||
|
||||
When you use the code this way, it is your responsibility to check for error before using the
|
||||
When there is no error, the error code simdjson::SUCCESS is returned: it evaluates as false as a Boolean.
|
||||
We have several error codes to indicate errors, they all evaluate to true as a Boolean: your software should not generally not depend on exact
|
||||
error codes. We may change the error codes in future releases and the exact error codes could vary depending on your system.
|
||||
|
||||
When you use the code without exceptions, it is your responsibility to check for error before using the
|
||||
result: if there is an error, the result value will not be valid and using it will caused undefined
|
||||
behavior.
|
||||
|
||||
@@ -318,6 +322,9 @@ int main(void) {
|
||||
}
|
||||
```
|
||||
|
||||
The `at()` method has linear-time complexity: it should not be used to iterate
|
||||
over the content of an array.
|
||||
|
||||
### Error Handling Example
|
||||
|
||||
This is how the example in "Using the Parsed JSON" could be written using only error code checking:
|
||||
|
||||
@@ -50,9 +50,9 @@ Inspecting the Detected Implementation
|
||||
You can check what implementation is running with `active_implementation`:
|
||||
|
||||
```c++
|
||||
cout << "simdjson v" << STRINGIFY(SIMDJSON_VERSION) << endl;
|
||||
cout << "Detected the best implementation for your machine: " << simdjson::active_implementation->name();
|
||||
cout << "(" << simdjson::active_implementation->description() << ")" << endl;
|
||||
cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << endl;
|
||||
cout << "Detected the best implementation for your machine: " << simdjson::get_active_implementation()->name();
|
||||
cout << "(" << simdjson::get_active_implementation()->description() << ")" << endl;
|
||||
```
|
||||
|
||||
Implementation detection will happen in this case when you first call `name()`.
|
||||
@@ -63,7 +63,7 @@ Querying Available Implementations
|
||||
You can list all available implementations, regardless of which one was selected:
|
||||
|
||||
```c++
|
||||
for (auto implementation : simdjson::available_implementations) {
|
||||
for (auto implementation : simdjson::get_available_implementations()) {
|
||||
cout << implementation->name() << ": " << implementation->description() << endl;
|
||||
}
|
||||
```
|
||||
@@ -71,10 +71,10 @@ for (auto implementation : simdjson::available_implementations) {
|
||||
And look them up by name:
|
||||
|
||||
```c++
|
||||
cout << simdjson::available_implementations["fallback"]->description() << endl;
|
||||
cout << simdjson::get_available_implementations()["fallback"]->description() << endl;
|
||||
```
|
||||
Though the fallback implementation should always be available, others might be missing. When
|
||||
an implementation is not available, the bracket call `simdjson::available_implementations[name]`
|
||||
an implementation is not available, the bracket call `simdjson::get_available_implementations()[name]`
|
||||
will return the null pointer.
|
||||
|
||||
The available implementations have been compiled but may not necessarily be run safely on your system
|
||||
@@ -90,18 +90,18 @@ can select the CPU architecture yourself:
|
||||
|
||||
```c++
|
||||
// Use the fallback implementation, even though my machine is fast enough for anything
|
||||
simdjson::active_implementation = simdjson::available_implementations["fallback"];
|
||||
simdjson::get_active_implementation() = simdjson::get_available_implementations()["fallback"];
|
||||
```
|
||||
|
||||
You are responsible for ensuring that the requirements of the selected implementation match your current system.
|
||||
Furthermore, you should check that the implementation is available before setting it to `simdjson::active_implementation`
|
||||
Furthermore, you should check that the implementation is available before setting it to `simdjson::get_active_implementation()`
|
||||
by comparing it with the null pointer.
|
||||
|
||||
```c++
|
||||
auto my_implementation = simdjson::available_implementations["haswell"];
|
||||
auto my_implementation = simdjson::get_available_implementations()["haswell"];
|
||||
if(! my_implementation) { exit(1); }
|
||||
if(! my_implementation->supported_by_runtime_system()) { exit(1); }
|
||||
simdjson::active_implementation = my_implementation;
|
||||
simdjson::get_active_implementation() = my_implementation;
|
||||
```
|
||||
|
||||
Checking that an Implementation can Run on your System
|
||||
@@ -110,12 +110,12 @@ Checking that an Implementation can Run on your System
|
||||
You should call `supported_by_runtime_system()` to compare the processor's features with the need of the implementation.
|
||||
|
||||
```c++
|
||||
for (auto implementation : simdjson::available_implementations) {
|
||||
for (auto implementation : simdjson::get_available_implementations()) {
|
||||
if(implementation->supported_by_runtime_system()) {
|
||||
cout << implementation->name() << ": " << implementation->description() << endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The call to `supported_by_runtime_system()` maybe relatively expensive. Do not call `supported_by_runtime_system()` each
|
||||
time you parse a JSON input (for example). It is meant to be called a handful of times at most in the life of a program.
|
||||
The call to `supported_by_runtime_system()` may be relatively expensive. Do not call `supported_by_runtime_system()` each
|
||||
time you parse a JSON input (for example). It is meant to be called a handful of times at most in the life of a program.
|
||||
|
||||
+3
-1
@@ -160,6 +160,8 @@ You should not expect the simdjson library to cause *downclocking* of your recen
|
||||
- [Whenever 512-bit AVX-512 instructions are used](https://lemire.me/blog/2018/09/07/avx-512-when-and-how-to-use-these-new-instructions/).
|
||||
- Whenever heavy 256-bit or wider instructions are used. Heavy instructions are those involving floating point operations or integer multiplications (since these execute on the floating point unit).
|
||||
|
||||
The simdjson library does not currently support AVX-512 instructions and it does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors.
|
||||
The simdjson library does not make use of heavy 256-bit instructions. We do use vectorized multiplications, but only using 128-bit registers. Thus there should be no downclocking due to simdjson on recent processors, except when AVX-512 is allowed and
|
||||
detected. However, we only allow AVX-512 on recent processors (Ice Lake/Tiger Lake or better) where [little to no frequency throttling is expected](https://travisdowns.github.io/blog/2020/08/19/icl-avx512-freq.html). If you can still concerned, you can easily disable AVX-512 with the CMake option `SIMDJSON_AVX512_ALLOWED` set to `OFF` (e.g., `cmake -D SIMDJSON_AVX512_ALLOWED=OFF -B build && cmake --build build`) or by setting
|
||||
the macro `SIMDJSON_AVX512_ALLOWED` to `0` in C++ prior to importing the headers.
|
||||
|
||||
You may still be worried about which SIMD instruction set is used by simdjson. Thankfully, [you can always determine and change which architecture-specific implementation is used](implementation-selection.md) by simdjson. Thus even if your CPU supports AVX2, you do not need to use AVX2. You are in control.
|
||||
|
||||
@@ -93,7 +93,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
|
||||
std::size_t nerrors=0;
|
||||
for(std::size_t i=0; i<Nimplementations; ++i) {
|
||||
auto& e=implementations[i];
|
||||
simdjson::active_implementation=e.impl;
|
||||
simdjson::get_active_implementation()=e.impl;
|
||||
e.error=e.parser.parse(Data,Size).get(e.element);
|
||||
if(e.error) {
|
||||
++nerrors;
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
std::vector<const simdjson::implementation*>
|
||||
get_runtime_supported_implementations() {
|
||||
std::vector<const simdjson::implementation*> ret;
|
||||
for(auto& e: simdjson::available_implementations) {
|
||||
for(auto& e: simdjson::get_available_implementations()) {
|
||||
if(e->supported_by_runtime_system()) {
|
||||
ret.emplace_back(e);
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
}
|
||||
*/
|
||||
|
||||
#include "simdjson/simdjson_version.h"
|
||||
#include "simdjson/dom.h"
|
||||
#include "simdjson/builtin.h"
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
unsigned long ret;
|
||||
@@ -73,7 +73,7 @@ simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) {
|
||||
* greating or equal to 63 in which case we trigger undefined behavior, but the output
|
||||
* of such undefined behavior is never used.
|
||||
**/
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) {
|
||||
return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes);
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x
|
||||
simdjson_really_inline simd8<T>& operator&=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast & other; return *this_cast; }
|
||||
simdjson_really_inline simd8<T>& operator^=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const { return vceqq_u8(*this, other); }
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return vceqq_u8(lhs, rhs); }
|
||||
|
||||
template<int N=1>
|
||||
simdjson_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
|
||||
// Determine the best builtin implementation
|
||||
#ifndef SIMDJSON_BUILTIN_IMPLEMENTATION
|
||||
#if SIMDJSON_CAN_ALWAYS_RUN_HASWELL
|
||||
#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION icelake
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_HASWELL
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION haswell
|
||||
#elif SIMDJSON_CAN_ALWAYS_RUN_WESTMERE
|
||||
#define SIMDJSON_BUILTIN_IMPLEMENTATION westmere
|
||||
|
||||
@@ -176,7 +176,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
|
||||
if (implementation) {
|
||||
err = implementation->allocate(capacity, max_depth);
|
||||
} else {
|
||||
err = simdjson::active_implementation->create_dom_parser_implementation(capacity, max_depth, implementation);
|
||||
err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation);
|
||||
}
|
||||
if (err) { return err; }
|
||||
return SUCCESS;
|
||||
|
||||
@@ -7,7 +7,12 @@
|
||||
namespace simdjson {
|
||||
|
||||
/**
|
||||
* All possible errors returned by simdjson.
|
||||
* All possible errors returned by simdjson. These error codes are subject to change
|
||||
* and not all simdjson kernel returns the same error code given the same input: it is not
|
||||
* well defined which error a given input should produce.
|
||||
*
|
||||
* Only SUCCESS evaluates to false as a Boolean. All other error codes will evaluate
|
||||
* to true as a Boolean.
|
||||
*/
|
||||
enum error_code {
|
||||
SUCCESS = 0, ///< No error
|
||||
|
||||
@@ -361,7 +361,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
||||
}
|
||||
|
||||
template<typename I>
|
||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
||||
if (digit > 9) {
|
||||
|
||||
@@ -104,7 +104,7 @@ public:
|
||||
|
||||
/**
|
||||
* Get the value at the given index. This function has linear-time complexity.
|
||||
* This function should only be called once as the array iterator is not reset between each call.
|
||||
* This function should only be called once on an array instance since the array iterator is not reset between each call.
|
||||
*
|
||||
* @return The value at the given index, or:
|
||||
* - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
|
||||
|
||||
@@ -94,6 +94,8 @@ public:
|
||||
*
|
||||
* The string is guaranteed to be valid UTF-8.
|
||||
*
|
||||
* Important: Calling get_string() twice on the same document is an error.
|
||||
*
|
||||
* @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
|
||||
* time it parses a document or when it is destroyed.
|
||||
* @returns INCORRECT_TYPE if the JSON value is not a string.
|
||||
@@ -267,7 +269,7 @@ public:
|
||||
simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
|
||||
/**
|
||||
* Get the value at the given index in the array. This function has linear-time complexity.
|
||||
* This function should only be called once as the array iterator is not reset between each call.
|
||||
* This function should only be called once on an array instance since the array iterator is not reset between each call.
|
||||
*
|
||||
* @return The value at the given index, or:
|
||||
* - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
|
||||
@@ -311,6 +313,10 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to
|
||||
* a key a single time. Doing object["mykey"].to_string()and then again object["mykey"].to_string()
|
||||
* is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -341,6 +347,10 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to a key
|
||||
* a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string()
|
||||
* is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -513,7 +523,7 @@ public:
|
||||
simdjson_really_inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
|
||||
/**
|
||||
* Consumes the document and returns a string_view instance corresponding to the
|
||||
* document as represented in JSON. It points inside the original byte array containg
|
||||
* document as represented in JSON. It points inside the original byte array containing
|
||||
* the JSON document.
|
||||
*/
|
||||
simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
|
||||
|
||||
@@ -111,7 +111,6 @@ protected:
|
||||
uint64_t unsigned_integer;
|
||||
} payload{0};
|
||||
number_type type{number_type::signed_integer};
|
||||
friend class value_iterator;
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -44,6 +44,10 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to a
|
||||
* key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string()
|
||||
* is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -77,6 +81,9 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to a key
|
||||
* a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
|
||||
@@ -21,7 +21,7 @@ simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t n
|
||||
SIMDJSON_TRY( implementation->set_capacity(new_capacity) );
|
||||
SIMDJSON_TRY( implementation->set_max_depth(new_max_depth) );
|
||||
} else {
|
||||
SIMDJSON_TRY( simdjson::active_implementation->create_dom_parser_implementation(new_capacity, new_max_depth, implementation) );
|
||||
SIMDJSON_TRY( simdjson::get_active_implementation()->create_dom_parser_implementation(new_capacity, new_max_depth, implementation) );
|
||||
}
|
||||
_capacity = new_capacity;
|
||||
_max_depth = new_max_depth;
|
||||
|
||||
@@ -182,7 +182,7 @@ public:
|
||||
* The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8).
|
||||
*
|
||||
* The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
|
||||
* Setting batch_size to excessively large or excesively small values may impact negatively the
|
||||
* Setting batch_size to excessively large or excessively small values may impact negatively the
|
||||
* performance.
|
||||
*
|
||||
* ### REQUIRED: Buffer Padding
|
||||
|
||||
@@ -99,6 +99,7 @@ inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON
|
||||
}
|
||||
} // namespace simdjson
|
||||
|
||||
namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace ondemand {
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTATION::ondemand::value x) {
|
||||
@@ -216,3 +217,4 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTA
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}}} // namespace simdjson::SIMDJSON_IMPLEMENTATION::ondemand
|
||||
@@ -29,6 +29,14 @@ inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON
|
||||
inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array> x);
|
||||
} // namespace simdjson
|
||||
|
||||
/**
|
||||
* We want to support argument-dependent lookup (ADL).
|
||||
* Hence we should define operator<< in the namespace
|
||||
* where the argument (here value, object, etc.) resides.
|
||||
* Credit: @madhur4127
|
||||
* See https://github.com/simdjson/simdjson/issues/1768
|
||||
*/
|
||||
namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace ondemand {
|
||||
|
||||
/**
|
||||
* Print JSON to an output stream.
|
||||
@@ -78,3 +86,4 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_IMPLEMENTA
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_IMPLEMENTATION::ondemand::object> x);
|
||||
#endif
|
||||
}}} // namespace simdjson::SIMDJSON_IMPLEMENTATION::ondemand
|
||||
@@ -62,6 +62,7 @@ template<> simdjson_really_inline simdjson_result<array> value::get() noexcept {
|
||||
template<> simdjson_really_inline simdjson_result<object> value::get() noexcept { return get_object(); }
|
||||
template<> simdjson_really_inline simdjson_result<raw_json_string> value::get() noexcept { return get_raw_json_string(); }
|
||||
template<> simdjson_really_inline simdjson_result<std::string_view> value::get() noexcept { return get_string(); }
|
||||
template<> simdjson_really_inline simdjson_result<number> value::get() noexcept { return get_number(); }
|
||||
template<> simdjson_really_inline simdjson_result<double> value::get() noexcept { return get_double(); }
|
||||
template<> simdjson_really_inline simdjson_result<uint64_t> value::get() noexcept { return get_uint64(); }
|
||||
template<> simdjson_really_inline simdjson_result<int64_t> value::get() noexcept { return get_int64(); }
|
||||
|
||||
@@ -121,6 +121,9 @@ public:
|
||||
*
|
||||
* Equivalent to get<std::string_view>().
|
||||
*
|
||||
* Important: a value should be consumed once. Calling get_string() twice on the same value
|
||||
* is an error.
|
||||
*
|
||||
* @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
|
||||
* time it parses a document or when it is destroyed.
|
||||
* @returns INCORRECT_TYPE if the JSON value is not a string.
|
||||
@@ -261,7 +264,7 @@ public:
|
||||
simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
|
||||
/**
|
||||
* Get the value at the given index in the array. This function has linear-time complexity.
|
||||
* This function should only be called once as the array iterator is not reset between each call.
|
||||
* This function should only be called once on an array instance since the array iterator is not reset between each call.
|
||||
*
|
||||
* @return The value at the given index, or:
|
||||
* - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
return (int)_tzcnt_u64(input_num);
|
||||
|
||||
@@ -41,11 +41,7 @@
|
||||
// has it as a macro.
|
||||
#ifndef _blsr_u64
|
||||
// we roll our own
|
||||
SIMDJSON_TARGET_HASWELL
|
||||
static simdjson_really_inline uint64_t _blsr_u64(uint64_t n) {
|
||||
return (n - 1) & n;
|
||||
}
|
||||
SIMDJSON_UNTARGET_HASWELL
|
||||
#define _blsr_u64(n) ((n - 1) & n)
|
||||
#endif // _blsr_u64
|
||||
#endif // SIMDJSON_CLANG_VISUAL_STUDIO
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ namespace simd {
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m256i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const { return _mm256_cmpeq_epi8(*this, other); }
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return _mm256_cmpeq_epi8(lhs, rhs); }
|
||||
|
||||
static const int SIZE = sizeof(base<T>::value);
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
#ifndef SIMDJSON_ICELAKE_H
|
||||
#define SIMDJSON_ICELAKE_H
|
||||
|
||||
#include "simdjson/implementation-base.h"
|
||||
|
||||
#if SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
|
||||
#if SIMDJSON_CAN_ALWAYS_RUN_ICELAKE
|
||||
#define SIMDJSON_TARGET_ICELAKE
|
||||
#define SIMDJSON_UNTARGET_ICELAKE
|
||||
#else
|
||||
#define SIMDJSON_TARGET_ICELAKE SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt")
|
||||
#define SIMDJSON_UNTARGET_ICELAKE SIMDJSON_UNTARGET_REGION
|
||||
#endif
|
||||
|
||||
namespace simdjson {
|
||||
/**
|
||||
* Implementation for Icelake (Intel AVX512).
|
||||
*/
|
||||
namespace icelake {
|
||||
} // namespace icelake
|
||||
} // namespace simdjson
|
||||
|
||||
//
|
||||
// These two need to be included outside SIMDJSON_TARGET_ICELAKE
|
||||
//
|
||||
#include "simdjson/icelake/implementation.h"
|
||||
#include "simdjson/icelake/intrinsics.h"
|
||||
|
||||
//
|
||||
// The rest need to be inside the region
|
||||
//
|
||||
#include "simdjson/icelake/begin.h"
|
||||
|
||||
// Declarations
|
||||
#include "simdjson/generic/dom_parser_implementation.h"
|
||||
#include "simdjson/icelake/bitmanipulation.h"
|
||||
#include "simdjson/icelake/bitmask.h"
|
||||
#include "simdjson/icelake/simd.h"
|
||||
#include "simdjson/generic/jsoncharutils.h"
|
||||
#include "simdjson/generic/atomparsing.h"
|
||||
#include "simdjson/icelake/stringparsing.h"
|
||||
#include "simdjson/icelake/numberparsing.h"
|
||||
#include "simdjson/icelake/end.h"
|
||||
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
#endif // SIMDJSON_ICELAKE_H
|
||||
@@ -0,0 +1,2 @@
|
||||
#define SIMDJSON_IMPLEMENTATION icelake
|
||||
SIMDJSON_TARGET_ICELAKE
|
||||
@@ -0,0 +1,61 @@
|
||||
#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H
|
||||
#define SIMDJSON_ICELAKE_BITMANIPULATION_H
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
return (int)_tzcnt_u64(input_num);
|
||||
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
////////
|
||||
// You might expect the next line to be equivalent to
|
||||
// return (int)_tzcnt_u64(input_num);
|
||||
// but the generated code differs and might be less efficient?
|
||||
////////
|
||||
return __builtin_ctzll(input_num);
|
||||
#endif // SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
}
|
||||
|
||||
/* result might be undefined when input_num is zero */
|
||||
simdjson_really_inline uint64_t clear_lowest_bit(uint64_t input_num) {
|
||||
return _blsr_u64(input_num);
|
||||
}
|
||||
|
||||
/* result might be undefined when input_num is zero */
|
||||
simdjson_really_inline int leading_zeroes(uint64_t input_num) {
|
||||
return int(_lzcnt_u64(input_num));
|
||||
}
|
||||
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
simdjson_really_inline unsigned __int64 count_ones(uint64_t input_num) {
|
||||
// note: we do not support legacy 32-bit Windows
|
||||
return __popcnt64(input_num);// Visual Studio wants two underscores
|
||||
}
|
||||
#else
|
||||
simdjson_really_inline long long int count_ones(uint64_t input_num) {
|
||||
return _popcnt64(input_num);
|
||||
}
|
||||
#endif
|
||||
|
||||
simdjson_really_inline bool add_overflow(uint64_t value1, uint64_t value2,
|
||||
uint64_t *result) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
return _addcarry_u64(0, value1, value2,
|
||||
reinterpret_cast<unsigned __int64 *>(result));
|
||||
#else
|
||||
return __builtin_uaddll_overflow(value1, value2,
|
||||
reinterpret_cast<unsigned long long *>(result));
|
||||
#endif
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H
|
||||
@@ -0,0 +1,25 @@
|
||||
#ifndef SIMDJSON_ICELAKE_BITMASK_H
|
||||
#define SIMDJSON_ICELAKE_BITMASK_H
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
//
|
||||
// Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered.
|
||||
//
|
||||
// For example, prefix_xor(00100100) == 00011100
|
||||
//
|
||||
simdjson_really_inline uint64_t prefix_xor(const uint64_t bitmask) {
|
||||
// There should be no such thing with a processor supporting avx2
|
||||
// but not clmul.
|
||||
__m128i all_ones = _mm_set1_epi8('\xFF');
|
||||
__m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0);
|
||||
return _mm_cvtsi128_si64(result);
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_BITMASK_H
|
||||
@@ -0,0 +1,2 @@
|
||||
SIMDJSON_UNTARGET_ICELAKE
|
||||
#undef SIMDJSON_IMPLEMENTATION
|
||||
@@ -0,0 +1,31 @@
|
||||
#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H
|
||||
#define SIMDJSON_ICELAKE_IMPLEMENTATION_H
|
||||
|
||||
#include "simdjson/implementation.h"
|
||||
|
||||
// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE
|
||||
namespace simdjson {
|
||||
namespace icelake {
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
class implementation final : public simdjson::implementation {
|
||||
public:
|
||||
simdjson_really_inline implementation() : simdjson::implementation(
|
||||
"icelake",
|
||||
"Intel/AMD AVX512",
|
||||
internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2
|
||||
) {}
|
||||
simdjson_warn_unused error_code create_dom_parser_implementation(
|
||||
size_t capacity,
|
||||
size_t max_length,
|
||||
std::unique_ptr<internal::dom_parser_implementation>& dst
|
||||
) const noexcept final;
|
||||
simdjson_warn_unused error_code minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept final;
|
||||
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) const noexcept final;
|
||||
};
|
||||
|
||||
} // namespace icelake
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H
|
||||
@@ -0,0 +1,56 @@
|
||||
#ifndef SIMDJSON_ICELAKE_INTRINSICS_H
|
||||
#define SIMDJSON_ICELAKE_INTRINSICS_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
|
||||
#ifdef SIMDJSON_VISUAL_STUDIO
|
||||
// under clang within visual studio, this will include <x86intrin.h>
|
||||
#include <intrin.h> // visual studio or clang
|
||||
#else
|
||||
#include <x86intrin.h> // elsewhere
|
||||
#endif // SIMDJSON_VISUAL_STUDIO
|
||||
|
||||
#ifdef SIMDJSON_CLANG_VISUAL_STUDIO
|
||||
/**
|
||||
* You are not supposed, normally, to include these
|
||||
* headers directly. Instead you should either include intrin.h
|
||||
* or x86intrin.h. However, when compiling with clang
|
||||
* under Windows (i.e., when _MSC_VER is set), these headers
|
||||
* only get included *if* the corresponding features are detected
|
||||
* from macros:
|
||||
* e.g., if __AVX2__ is set... in turn, we normally set these
|
||||
* macros by compiling against the corresponding architecture
|
||||
* (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole
|
||||
* software with these advanced instructions. In simdjson, we
|
||||
* want to compile the whole program for a generic target,
|
||||
* and only target our specific kernels. As a workaround,
|
||||
* we directly include the needed headers. These headers would
|
||||
* normally guard against such usage, but we carefully included
|
||||
* <x86intrin.h> (or <intrin.h>) before, so the headers
|
||||
* are fooled.
|
||||
*/
|
||||
#include <bmiintrin.h> // for _blsr_u64
|
||||
#include <lzcntintrin.h> // for __lzcnt64
|
||||
#include <immintrin.h> // for most things (AVX2, AVX512, _popcnt64)
|
||||
#include <smmintrin.h>
|
||||
#include <tmmintrin.h>
|
||||
#include <avxintrin.h>
|
||||
#include <avx2intrin.h>
|
||||
#include <wmmintrin.h> // for _mm_clmulepi64_si128
|
||||
// Important: we need the AVX-512 headers:
|
||||
#include <avx512fintrin.h>
|
||||
#include <avx512dqintrin.h>
|
||||
#include <avx512cdintrin.h>
|
||||
#include <avx512bwintrin.h>
|
||||
#include <avx512vlintrin.h>
|
||||
#include <avx512vbmiintrin.h>
|
||||
#include <avx512vbmi2intrin.h>
|
||||
// unfortunately, we may not get _blsr_u64, but, thankfully, clang
|
||||
// has it as a macro.
|
||||
#ifndef _blsr_u64
|
||||
// we roll our own
|
||||
#define _blsr_u64(n) ((n - 1) & n)
|
||||
#endif // _blsr_u64
|
||||
#endif // SIMDJSON_CLANG_VISUAL_STUDIO
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_INTRINSICS_H
|
||||
@@ -0,0 +1,34 @@
|
||||
#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_H
|
||||
#define SIMDJSON_ICELAKE_NUMBERPARSING_H
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
// this actually computes *16* values so we are being wasteful.
|
||||
const __m128i ascii0 = _mm_set1_epi8('0');
|
||||
const __m128i mul_1_10 =
|
||||
_mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1);
|
||||
const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
|
||||
const __m128i mul_1_10000 =
|
||||
_mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
|
||||
const __m128i input = _mm_sub_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
|
||||
const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
|
||||
const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
|
||||
const __m128i t3 = _mm_packus_epi32(t2, t2);
|
||||
const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000);
|
||||
return _mm_cvtsi128_si32(
|
||||
t4); // only captures the sum of the first 8 digits, drop the rest
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||
|
||||
#include "simdjson/generic/numberparsing.h"
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_NUMBERPARSING_H
|
||||
@@ -0,0 +1,341 @@
|
||||
#ifndef SIMDJSON_ICELAKE_SIMD_H
|
||||
#define SIMDJSON_ICELAKE_SIMD_H
|
||||
|
||||
#include "simdjson/internal/simdprune_tables.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
namespace simd {
|
||||
|
||||
// Forward-declared so they can be used by splat and friends.
|
||||
template<typename Child>
|
||||
struct base {
|
||||
__m512i value;
|
||||
|
||||
// Zero constructor
|
||||
simdjson_really_inline base() : value{__m512i()} {}
|
||||
|
||||
// Conversion from SIMD register
|
||||
simdjson_really_inline base(const __m512i _value) : value(_value) {}
|
||||
|
||||
// Conversion to SIMD register
|
||||
simdjson_really_inline operator const __m512i&() const { return this->value; }
|
||||
simdjson_really_inline operator __m512i&() { return this->value; }
|
||||
|
||||
// Bit operations
|
||||
simdjson_really_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); }
|
||||
simdjson_really_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); }
|
||||
simdjson_really_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); }
|
||||
simdjson_really_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); }
|
||||
simdjson_really_inline Child& operator|=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast | other; return *this_cast; }
|
||||
simdjson_really_inline Child& operator&=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast & other; return *this_cast; }
|
||||
simdjson_really_inline Child& operator^=(const Child other) { auto this_cast = static_cast<Child*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
|
||||
};
|
||||
|
||||
// Forward-declared so they can be used by splat and friends.
|
||||
template<typename T>
|
||||
struct simd8;
|
||||
|
||||
template<typename T, typename Mask=simd8<bool>>
|
||||
struct base8: base<simd8<T>> {
|
||||
typedef uint32_t bitmask_t;
|
||||
typedef uint64_t bitmask2_t;
|
||||
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m512i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline uint64_t operator==(const simd8<T> other) const { return _mm512_cmpeq_epi8_mask(*this, other); }
|
||||
|
||||
static const int SIZE = sizeof(base<T>::value);
|
||||
|
||||
template<int N=1>
|
||||
simdjson_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
|
||||
return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), 16 - N);
|
||||
}
|
||||
};
|
||||
|
||||
// SIMD byte mask type (returned by things like eq and gt)
|
||||
template<>
|
||||
struct simd8<bool>: base8<bool> {
|
||||
static simdjson_really_inline simd8<bool> splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); }
|
||||
|
||||
simdjson_really_inline simd8<bool>() : base8() {}
|
||||
simdjson_really_inline simd8<bool>(const __m512i _value) : base8<bool>(_value) {}
|
||||
// Splat constructor
|
||||
simdjson_really_inline simd8<bool>(bool _value) : base8<bool>(splat(_value)) {}
|
||||
simdjson_really_inline bool any() const { return !!_mm512_test_epi8_mask (*this, *this); }
|
||||
simdjson_really_inline simd8<bool> operator~() const { return *this ^ true; }
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct base8_numeric: base8<T> {
|
||||
static simdjson_really_inline simd8<T> splat(T _value) { return _mm512_set1_epi8(_value); }
|
||||
static simdjson_really_inline simd8<T> zero() { return _mm512_setzero_si512(); }
|
||||
static simdjson_really_inline simd8<T> load(const T values[64]) {
|
||||
return _mm512_loadu_si512(reinterpret_cast<const __m512i *>(values));
|
||||
}
|
||||
// Repeat 16 values as many times as necessary (usually for lookup tables)
|
||||
static simdjson_really_inline simd8<T> repeat_16(
|
||||
T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7,
|
||||
T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15
|
||||
) {
|
||||
return simd8<T>(
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15
|
||||
);
|
||||
}
|
||||
|
||||
simdjson_really_inline base8_numeric() : base8<T>() {}
|
||||
simdjson_really_inline base8_numeric(const __m512i _value) : base8<T>(_value) {}
|
||||
|
||||
// Store to array
|
||||
simdjson_really_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i *>(dst), *this); }
|
||||
|
||||
// Addition/subtraction are the same for signed and unsigned
|
||||
simdjson_really_inline simd8<T> operator+(const simd8<T> other) const { return _mm512_add_epi8(*this, other); }
|
||||
simdjson_really_inline simd8<T> operator-(const simd8<T> other) const { return _mm512_sub_epi8(*this, other); }
|
||||
simdjson_really_inline simd8<T>& operator+=(const simd8<T> other) { *this = *this + other; return *static_cast<simd8<T>*>(this); }
|
||||
simdjson_really_inline simd8<T>& operator-=(const simd8<T> other) { *this = *this - other; return *static_cast<simd8<T>*>(this); }
|
||||
|
||||
// Override to distinguish from bool version
|
||||
simdjson_really_inline simd8<T> operator~() const { return *this ^ 0xFFu; }
|
||||
|
||||
// Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values)
|
||||
template<typename L>
|
||||
simdjson_really_inline simd8<L> lookup_16(simd8<L> lookup_table) const {
|
||||
return _mm512_shuffle_epi8(lookup_table, *this);
|
||||
}
|
||||
|
||||
// Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset).
|
||||
// Passing a 0 value for mask would be equivalent to writing out every byte to output.
|
||||
// Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes
|
||||
// get written.
|
||||
// Design consideration: it seems like a function with the
|
||||
// signature simd8<L> compress(uint32_t mask) would be
|
||||
// sensible, but the AVX ISA makes this kind of approach difficult.
|
||||
template<typename L>
|
||||
simdjson_really_inline void compress(uint64_t mask, L * output) const {
|
||||
_mm512_mask_compressstoreu_epi8 (output,~mask,*this);
|
||||
}
|
||||
|
||||
template<typename L>
|
||||
simdjson_really_inline simd8<L> lookup_16(
|
||||
L replace0, L replace1, L replace2, L replace3,
|
||||
L replace4, L replace5, L replace6, L replace7,
|
||||
L replace8, L replace9, L replace10, L replace11,
|
||||
L replace12, L replace13, L replace14, L replace15) const {
|
||||
return lookup_16(simd8<L>::repeat_16(
|
||||
replace0, replace1, replace2, replace3,
|
||||
replace4, replace5, replace6, replace7,
|
||||
replace8, replace9, replace10, replace11,
|
||||
replace12, replace13, replace14, replace15
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Signed bytes
|
||||
template<>
|
||||
struct simd8<int8_t> : base8_numeric<int8_t> {
|
||||
simdjson_really_inline simd8() : base8_numeric<int8_t>() {}
|
||||
simdjson_really_inline simd8(const __m512i _value) : base8_numeric<int8_t>(_value) {}
|
||||
// Splat constructor
|
||||
simdjson_really_inline simd8(int8_t _value) : simd8(splat(_value)) {}
|
||||
// Array constructor
|
||||
simdjson_really_inline simd8(const int8_t values[64]) : simd8(load(values)) {}
|
||||
// Member-by-member initialization
|
||||
simdjson_really_inline simd8(
|
||||
int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7,
|
||||
int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15,
|
||||
int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23,
|
||||
int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31,
|
||||
int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39,
|
||||
int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47,
|
||||
int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55,
|
||||
int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63
|
||||
) : simd8(_mm512_set_epi8(
|
||||
v63, v62, v61, v60, v59, v58, v57, v56,
|
||||
v55, v54, v53, v52, v51, v50, v49, v48,
|
||||
v47, v46, v45, v44, v43, v42, v41, v40,
|
||||
v39, v38, v37, v36, v35, v34, v33, v32,
|
||||
v31, v30, v29, v28, v27, v26, v25, v24,
|
||||
v23, v22, v21, v20, v19, v18, v17, v16,
|
||||
v15, v14, v13, v12, v11, v10, v9, v8,
|
||||
v7, v6, v5, v4, v3, v2, v1, v0
|
||||
)) {}
|
||||
|
||||
// Repeat 16 values as many times as necessary (usually for lookup tables)
|
||||
simdjson_really_inline static simd8<int8_t> repeat_16(
|
||||
int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7,
|
||||
int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15
|
||||
) {
|
||||
return simd8<int8_t>(
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15
|
||||
);
|
||||
}
|
||||
|
||||
// Order-sensitive comparisons
|
||||
simdjson_really_inline simd8<int8_t> max_val(const simd8<int8_t> other) const { return _mm512_max_epi8(*this, other); }
|
||||
simdjson_really_inline simd8<int8_t> min_val(const simd8<int8_t> other) const { return _mm512_min_epi8(*this, other); }
|
||||
|
||||
simdjson_really_inline simd8<bool> operator>(const simd8<int8_t> other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other),_mm512_set1_epi8(uint8_t(0x80))); }
|
||||
simdjson_really_inline simd8<bool> operator<(const simd8<int8_t> other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this),_mm512_set1_epi8(uint8_t(0x80))); }
|
||||
};
|
||||
|
||||
// Unsigned bytes
|
||||
template<>
|
||||
struct simd8<uint8_t>: base8_numeric<uint8_t> {
|
||||
simdjson_really_inline simd8() : base8_numeric<uint8_t>() {}
|
||||
simdjson_really_inline simd8(const __m512i _value) : base8_numeric<uint8_t>(_value) {}
|
||||
// Splat constructor
|
||||
simdjson_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {}
|
||||
// Array constructor
|
||||
simdjson_really_inline simd8(const uint8_t values[64]) : simd8(load(values)) {}
|
||||
// Member-by-member initialization
|
||||
simdjson_really_inline simd8(
|
||||
uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7,
|
||||
uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15,
|
||||
uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23,
|
||||
uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31,
|
||||
uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39,
|
||||
uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47,
|
||||
uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55,
|
||||
uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63
|
||||
) : simd8(_mm512_set_epi8(
|
||||
v63, v62, v61, v60, v59, v58, v57, v56,
|
||||
v55, v54, v53, v52, v51, v50, v49, v48,
|
||||
v47, v46, v45, v44, v43, v42, v41, v40,
|
||||
v39, v38, v37, v36, v35, v34, v33, v32,
|
||||
v31, v30, v29, v28, v27, v26, v25, v24,
|
||||
v23, v22, v21, v20, v19, v18, v17, v16,
|
||||
v15, v14, v13, v12, v11, v10, v9, v8,
|
||||
v7, v6, v5, v4, v3, v2, v1, v0
|
||||
)) {}
|
||||
|
||||
// Repeat 16 values as many times as necessary (usually for lookup tables)
|
||||
simdjson_really_inline static simd8<uint8_t> repeat_16(
|
||||
uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7,
|
||||
uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15
|
||||
) {
|
||||
return simd8<uint8_t>(
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15,
|
||||
v0, v1, v2, v3, v4, v5, v6, v7,
|
||||
v8, v9, v10,v11,v12,v13,v14,v15
|
||||
);
|
||||
}
|
||||
|
||||
// Saturated math
|
||||
simdjson_really_inline simd8<uint8_t> saturating_add(const simd8<uint8_t> other) const { return _mm512_adds_epu8(*this, other); }
|
||||
simdjson_really_inline simd8<uint8_t> saturating_sub(const simd8<uint8_t> other) const { return _mm512_subs_epu8(*this, other); }
|
||||
|
||||
// Order-specific operations
|
||||
simdjson_really_inline simd8<uint8_t> max_val(const simd8<uint8_t> other) const { return _mm512_max_epu8(*this, other); }
|
||||
simdjson_really_inline simd8<uint8_t> min_val(const simd8<uint8_t> other) const { return _mm512_min_epu8(other, *this); }
|
||||
// Same as >, but only guarantees true is nonzero (< guarantees true = -1)
|
||||
simdjson_really_inline simd8<uint8_t> gt_bits(const simd8<uint8_t> other) const { return this->saturating_sub(other); }
|
||||
// Same as <, but only guarantees true is nonzero (< guarantees true = -1)
|
||||
simdjson_really_inline simd8<uint8_t> lt_bits(const simd8<uint8_t> other) const { return other.saturating_sub(*this); }
|
||||
simdjson_really_inline uint64_t operator<=(const simd8<uint8_t> other) const { return other.max_val(*this) == other; }
|
||||
simdjson_really_inline uint64_t operator>=(const simd8<uint8_t> other) const { return other.min_val(*this) == other; }
|
||||
simdjson_really_inline simd8<bool> operator>(const simd8<uint8_t> other) const { return this->gt_bits(other).any_bits_set(); }
|
||||
simdjson_really_inline simd8<bool> operator<(const simd8<uint8_t> other) const { return this->lt_bits(other).any_bits_set(); }
|
||||
|
||||
// Bit-specific operations
|
||||
simdjson_really_inline simd8<bool> bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); }
|
||||
simdjson_really_inline simd8<bool> bits_not_set(simd8<uint8_t> bits) const { return (*this & bits).bits_not_set(); }
|
||||
simdjson_really_inline simd8<bool> any_bits_set() const { return ~this->bits_not_set(); }
|
||||
simdjson_really_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return ~this->bits_not_set(bits); }
|
||||
|
||||
simdjson_really_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; }
|
||||
simdjson_really_inline bool bits_not_set_anywhere() const {
|
||||
return !_mm512_test_epi8_mask(*this, *this);
|
||||
}
|
||||
simdjson_really_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); }
|
||||
simdjson_really_inline bool bits_not_set_anywhere(simd8<uint8_t> bits) const { return !_mm512_test_epi8_mask(*this, bits); }
|
||||
simdjson_really_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return !bits_not_set_anywhere(bits); }
|
||||
template<int N>
|
||||
simdjson_really_inline simd8<uint8_t> shr() const { return simd8<uint8_t>(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); }
|
||||
template<int N>
|
||||
simdjson_really_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); }
|
||||
// Get one of the bits and make a bitmask out of it.
|
||||
// e.g. value.get_bit<7>() gets the high bit
|
||||
template<int N>
|
||||
simdjson_really_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); }
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
struct simd8x64 {
|
||||
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
|
||||
static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block.");
|
||||
const simd8<T> chunks[NUM_CHUNKS];
|
||||
|
||||
simd8x64(const simd8x64<T>& o) = delete; // no copy allowed
|
||||
simd8x64<T>& operator=(const simd8<T>& other) = delete; // no assignment allowed
|
||||
simd8x64() = delete; // no default constructor allowed
|
||||
|
||||
simdjson_really_inline simd8x64(const simd8<T> chunk0, const simd8<T> chunk1) : chunks{chunk0, chunk1} {}
|
||||
simdjson_really_inline simd8x64(const simd8<T> chunk0) : chunks{chunk0} {}
|
||||
simdjson_really_inline simd8x64(const T ptr[64]) : chunks{simd8<T>::load(ptr)} {}
|
||||
|
||||
simdjson_really_inline uint64_t compress(uint64_t mask, T * output) const {
|
||||
this->chunks[0].compress(mask, output);
|
||||
return 64 - count_ones(mask);
|
||||
}
|
||||
|
||||
simdjson_really_inline void store(T ptr[64]) const {
|
||||
this->chunks[0].store(ptr+sizeof(simd8<T>)*0);
|
||||
}
|
||||
|
||||
simdjson_really_inline simd8<T> reduce_or() const {
|
||||
return this->chunks[0];
|
||||
}
|
||||
|
||||
simdjson_really_inline simd8x64<T> bit_or(const T m) const {
|
||||
const simd8<T> mask = simd8<T>::splat(m);
|
||||
return simd8x64<T>(
|
||||
this->chunks[0] | mask
|
||||
);
|
||||
}
|
||||
|
||||
simdjson_really_inline uint64_t eq(const T m) const {
|
||||
const simd8<T> mask = simd8<T>::splat(m);
|
||||
return this->chunks[0] == mask;
|
||||
}
|
||||
|
||||
simdjson_really_inline uint64_t eq(const simd8x64<uint8_t> &other) const {
|
||||
return this->chunks[0] == other.chunks[0];
|
||||
}
|
||||
|
||||
simdjson_really_inline uint64_t lteq(const T m) const {
|
||||
const simd8<T> mask = simd8<T>::splat(m);
|
||||
return this->chunks[0] <= mask;
|
||||
}
|
||||
simdjson_really_inline operator __m512i() const { return __m512i(this->chunks[0]); }
|
||||
|
||||
}; // struct simd8x64<T>
|
||||
|
||||
} // namespace simd
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_SIMD_H
|
||||
@@ -0,0 +1,48 @@
|
||||
#ifndef SIMDJSON_ICELAKE_STRINGPARSING_H
|
||||
#define SIMDJSON_ICELAKE_STRINGPARSING_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include "simdjson/icelake/simd.h"
|
||||
#include "simdjson/icelake/bitmanipulation.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
using namespace simd;
|
||||
|
||||
// Holds backslashes and quotes locations.
|
||||
struct backslash_and_quote {
|
||||
public:
|
||||
static constexpr uint32_t BYTES_PROCESSED = 32;
|
||||
simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
|
||||
|
||||
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
||||
simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; }
|
||||
simdjson_really_inline int quote_index() { return trailing_zeroes(quote_bits); }
|
||||
simdjson_really_inline int backslash_index() { return trailing_zeroes(bs_bits); }
|
||||
|
||||
uint64_t bs_bits;
|
||||
uint64_t quote_bits;
|
||||
}; // struct backslash_and_quote
|
||||
|
||||
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
||||
// this can read up to 15 bytes beyond the buffer size, but we require
|
||||
// SIMDJSON_PADDING of padding
|
||||
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
|
||||
simd8<uint8_t> v(src);
|
||||
// store to dest unconditionally - we can overwrite the bits we don't like later
|
||||
v.store(dst);
|
||||
return {
|
||||
static_cast<uint64_t>(v == '\\'), // bs_bits
|
||||
static_cast<uint64_t>(v == '"'), // quote_bits
|
||||
};
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#include "simdjson/generic/stringparsing.h"
|
||||
|
||||
#endif // SIMDJSON_ICELAKE_STRINGPARSING_H
|
||||
@@ -56,7 +56,7 @@ public:
|
||||
/**
|
||||
* The name of this implementation.
|
||||
*
|
||||
* const implementation *impl = simdjson::active_implementation;
|
||||
* const implementation *impl = simdjson::get_active_implementation();
|
||||
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
|
||||
*
|
||||
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
|
||||
@@ -66,7 +66,7 @@ public:
|
||||
/**
|
||||
* The description of this implementation.
|
||||
*
|
||||
* const implementation *impl = simdjson::active_implementation;
|
||||
* const implementation *impl = simdjson::get_active_implementation();
|
||||
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
|
||||
*
|
||||
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
|
||||
@@ -95,7 +95,7 @@ public:
|
||||
/**
|
||||
* @private For internal implementation use
|
||||
*
|
||||
* const implementation *impl = simdjson::active_implementation;
|
||||
* const implementation *impl = simdjson::get_active_implementation();
|
||||
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
|
||||
*
|
||||
* @param capacity The largest document that will be passed to the parser.
|
||||
@@ -189,10 +189,10 @@ public:
|
||||
*
|
||||
* Case sensitive.
|
||||
*
|
||||
* const implementation *impl = simdjson::available_implementations["westmere"];
|
||||
* const implementation *impl = simdjson::get_available_implementations()["westmere"];
|
||||
* if (!impl) { exit(1); }
|
||||
* if (!imp->supported_by_runtime_system()) { exit(1); }
|
||||
* simdjson::active_implementation = impl;
|
||||
* simdjson::get_active_implementation() = impl;
|
||||
*
|
||||
* @param name the implementation to find, e.g. "westmere", "haswell", "arm64"
|
||||
* @return the implementation, or nullptr if the parse failed.
|
||||
@@ -210,7 +210,7 @@ public:
|
||||
* This is used to initialize the implementation on startup.
|
||||
*
|
||||
* const implementation *impl = simdjson::available_implementation::detect_best_supported();
|
||||
* simdjson::active_implementation = impl;
|
||||
* simdjson::get_active_implementation() = impl;
|
||||
*
|
||||
* @return the most advanced supported implementation for the current host, or an
|
||||
* implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported
|
||||
@@ -242,14 +242,14 @@ private:
|
||||
/**
|
||||
* The list of available implementations compiled into simdjson.
|
||||
*/
|
||||
extern SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations;
|
||||
extern SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations();
|
||||
|
||||
/**
|
||||
* The active implementation.
|
||||
*
|
||||
* Automatically initialized on first use to the most advanced implementation supported by this hardware.
|
||||
*/
|
||||
extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation;
|
||||
extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation();
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
|
||||
@@ -13,16 +13,42 @@
|
||||
#endif
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64
|
||||
|
||||
#ifdef __has_include
|
||||
// How do we detect that a compiler supports vbmi2?
|
||||
// For sure if the following header is found, we are ok?
|
||||
#if __has_include(<avx512vbmi2intrin.h>)
|
||||
#define SIMDJSON_COMPILER_SUPPORTS_VBMI2 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
// Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected
|
||||
// at runtime.
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
#define SIMDJSON_IMPLEMENTATION_ICELAKE ((SIMDJSON_IS_X86_64) && (SIMDJSON_AVX512_ALLOWED) && (SIMDJSON_COMPILER_SUPPORTS_VBMI2))
|
||||
#endif
|
||||
|
||||
#ifdef _MSC_VER
|
||||
// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see
|
||||
// https://github.com/simdjson/simdjson/issues/1247
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__))
|
||||
#else
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__))
|
||||
#endif
|
||||
|
||||
// Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected
|
||||
// at runtime.
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
#define SIMDJSON_IMPLEMENTATION_HASWELL (SIMDJSON_IS_X86_64)
|
||||
#define SIMDJSON_IMPLEMENTATION_HASWELL SIMDJSON_IS_X86_64
|
||||
#endif
|
||||
#ifdef _MSC_VER
|
||||
// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see
|
||||
// https://github.com/simdjson/simdjson/issues/1247
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__))
|
||||
#else
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__))
|
||||
#endif
|
||||
|
||||
// Default Westmere to on if this is x86-64, unless we'll always select Haswell.
|
||||
// Default Westmere to on if this is x86-64. Note that the macro SIMDJSON_REQUIRES_HASWELL appears unused.
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
#define SIMDJSON_IMPLEMENTATION_WESTMERE (SIMDJSON_IS_X86_64 && !SIMDJSON_REQUIRES_HASWELL)
|
||||
#endif
|
||||
@@ -45,6 +71,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
||||
// Implementations
|
||||
#include "simdjson/arm64.h"
|
||||
#include "simdjson/fallback.h"
|
||||
#include "simdjson/icelake.h"
|
||||
#include "simdjson/haswell.h"
|
||||
#include "simdjson/ppc64.h"
|
||||
#include "simdjson/westmere.h"
|
||||
|
||||
@@ -66,7 +66,16 @@ enum instruction_set {
|
||||
PCLMULQDQ = 0x10,
|
||||
BMI1 = 0x20,
|
||||
BMI2 = 0x40,
|
||||
ALTIVEC = 0x80
|
||||
ALTIVEC = 0x80,
|
||||
AVX512F = 0x100,
|
||||
AVX512DQ = 0x200,
|
||||
AVX512IFMA = 0x400,
|
||||
AVX512PF = 0x800,
|
||||
AVX512ER = 0x1000,
|
||||
AVX512CD = 0x2000,
|
||||
AVX512BW = 0x4000,
|
||||
AVX512VL = 0x8000,
|
||||
AVX512VBMI2 = 0x10000
|
||||
};
|
||||
|
||||
#if defined(__PPC64__)
|
||||
@@ -96,11 +105,20 @@ static inline uint32_t detect_supported_architectures() {
|
||||
|
||||
namespace {
|
||||
// Can be found on Intel ISA Reference for CPUID
|
||||
constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1
|
||||
constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1
|
||||
constexpr uint32_t cpuid_avx2_bit = 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_bmi1_bit = 1 << 3; ///< @private bit 3 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_bmi2_bit = 1 << 8; ///< @private bit 8 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512f_bit = 1 << 16; ///< @private bit 16 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512dq_bit = 1 << 17; ///< @private bit 17 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512ifma_bit = 1 << 21; ///< @private bit 21 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512pf_bit = 1 << 26; ///< @private bit 26 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512er_bit = 1 << 27; ///< @private bit 27 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512cd_bit = 1 << 28; ///< @private bit 28 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512bw_bit = 1 << 30; ///< @private bit 30 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512vl_bit = 1 << 31; ///< @private bit 31 of EBX for EAX=0x7
|
||||
constexpr uint32_t cpuid_avx512vbmi2_bit = 1 << 6; ///< @private bit 6 of ECX for EAX=0x7
|
||||
constexpr uint32_t cpuid_sse42_bit = 1 << 20; ///< @private bit 20 of ECX for EAX=0x1
|
||||
constexpr uint32_t cpuid_pclmulqdq_bit = 1 << 1; ///< @private bit 1 of ECX for EAX=0x1
|
||||
}
|
||||
|
||||
|
||||
@@ -146,6 +164,42 @@ static inline uint32_t detect_supported_architectures() {
|
||||
host_isa |= instruction_set::BMI2;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512f_bit) {
|
||||
host_isa |= instruction_set::AVX512F;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512dq_bit) {
|
||||
host_isa |= instruction_set::AVX512DQ;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512ifma_bit) {
|
||||
host_isa |= instruction_set::AVX512IFMA;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512pf_bit) {
|
||||
host_isa |= instruction_set::AVX512PF;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512er_bit) {
|
||||
host_isa |= instruction_set::AVX512ER;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512cd_bit) {
|
||||
host_isa |= instruction_set::AVX512CD;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512bw_bit) {
|
||||
host_isa |= instruction_set::AVX512BW;
|
||||
}
|
||||
|
||||
if (ebx & cpuid_avx512vl_bit) {
|
||||
host_isa |= instruction_set::AVX512VL;
|
||||
}
|
||||
|
||||
if (ecx & cpuid_avx512vbmi2_bit) {
|
||||
host_isa |= instruction_set::AVX512VBMI2;
|
||||
}
|
||||
|
||||
// EBX for EAX=0x1
|
||||
eax = 0x1;
|
||||
cpuid(&eax, &ebx, &ecx, &edx);
|
||||
|
||||
@@ -51,7 +51,7 @@ inline padded_string::padded_string(size_t length) noexcept
|
||||
}
|
||||
inline padded_string::padded_string(const char *data, size_t length) noexcept
|
||||
: viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) {
|
||||
if ((data != nullptr) and (data_ptr != nullptr)) {
|
||||
if ((data != nullptr) && (data_ptr != nullptr)) {
|
||||
std::memcpy(data_ptr, data, length);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,10 +71,10 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
|
||||
#endif // SIMDJSON_IS_32BITS
|
||||
|
||||
// this is almost standard?
|
||||
#undef STRINGIFY_IMPLEMENTATION_
|
||||
#undef STRINGIFY
|
||||
#define STRINGIFY_IMPLEMENTATION_(a) #a
|
||||
#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a)
|
||||
#undef SIMDJSON_STRINGIFY_IMPLEMENTATION_
|
||||
#undef SIMDJSON_STRINGIFY
|
||||
#define SIMDJSON_STRINGIFY_IMPLEMENTATION_(a) #a
|
||||
#define SIMDJSON_STRINGIFY(a) SIMDJSON_STRINGIFY_IMPLEMENTATION_(a)
|
||||
|
||||
// Our fast kernels require 64-bit systems.
|
||||
//
|
||||
@@ -98,13 +98,13 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
|
||||
// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a
|
||||
// namespace.
|
||||
#define SIMDJSON_TARGET_REGION(T) \
|
||||
_Pragma(STRINGIFY( \
|
||||
_Pragma(SIMDJSON_STRINGIFY( \
|
||||
clang attribute push(__attribute__((target(T))), apply_to = function)))
|
||||
#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop")
|
||||
#elif defined(__GNUC__)
|
||||
// GCC is easier
|
||||
#define SIMDJSON_TARGET_REGION(T) \
|
||||
_Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))
|
||||
_Pragma("GCC push_options") _Pragma(SIMDJSON_STRINGIFY(GCC target(T)))
|
||||
#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options")
|
||||
#endif // clang then gcc
|
||||
|
||||
@@ -138,11 +138,11 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
|
||||
|
||||
|
||||
#if defined(__clang__)
|
||||
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
|
||||
#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
|
||||
#elif defined(__GNUC__)
|
||||
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
|
||||
#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
|
||||
#else
|
||||
#define NO_SANITIZE_UNDEFINED
|
||||
#define SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
#endif
|
||||
|
||||
#ifdef SIMDJSON_VISUAL_STUDIO
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
unsigned long ret;
|
||||
|
||||
@@ -1,7 +1,11 @@
|
||||
#ifndef SIMDJSON_PPC64_NUMBERPARSING_H
|
||||
#define SIMDJSON_PPC64_NUMBERPARSING_H
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <byteswap.h>
|
||||
#elif defined(__FreeBSD__)
|
||||
#include <sys/endian.h>
|
||||
#endif
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
@@ -14,7 +18,11 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
uint64_t val;
|
||||
std::memcpy(&val, chars, sizeof(uint64_t));
|
||||
#ifdef __BIG_ENDIAN__
|
||||
#if defined(__linux__)
|
||||
val = bswap_64(val);
|
||||
#elif defined(__FreeBSD__)
|
||||
val = bswap64(val);
|
||||
#endif
|
||||
#endif
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
|
||||
@@ -69,8 +69,8 @@ struct base8 : base<simd8<T>> {
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const {
|
||||
return (__m128i)vec_cmpeq(this->value, (__m128i)other);
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) {
|
||||
return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs);
|
||||
}
|
||||
|
||||
static const int SIZE = sizeof(base<simd8<T>>::value);
|
||||
|
||||
@@ -8,7 +8,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
unsigned long ret;
|
||||
|
||||
@@ -44,7 +44,7 @@ namespace simd {
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const { return _mm_cmpeq_epi8(*this, other); }
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return _mm_cmpeq_epi8(lhs, rhs); }
|
||||
|
||||
static const int SIZE = sizeof(base<simd8<T>>::value);
|
||||
|
||||
|
||||
+85
-58
@@ -1,4 +1,4 @@
|
||||
/* auto-generated on 2021-10-27 19:25:23 -0400. Do not edit! */
|
||||
/* auto-generated on 2022-01-31 11:38:54 -0500. Do not edit! */
|
||||
/* begin file src/simdjson.cpp */
|
||||
#include "simdjson.h"
|
||||
|
||||
@@ -2620,19 +2620,34 @@ namespace internal {
|
||||
// without requiring a static initializer.
|
||||
|
||||
#if SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
const haswell::implementation haswell_singleton{};
|
||||
static const haswell::implementation* get_haswell_singleton() {
|
||||
static const haswell::implementation haswell_singleton{};
|
||||
return &haswell_singleton;
|
||||
}
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
const westmere::implementation westmere_singleton{};
|
||||
static const westmere::implementation* get_westmere_singleton() {
|
||||
static const westmere::implementation westmere_singleton{};
|
||||
return &westmere_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64
|
||||
const arm64::implementation arm64_singleton{};
|
||||
static const arm64::implementation* get_arm64_singleton() {
|
||||
static const arm64::implementation arm64_singleton{};
|
||||
return &arm64_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ARM64
|
||||
#if SIMDJSON_IMPLEMENTATION_PPC64
|
||||
const ppc64::implementation ppc64_singleton{};
|
||||
static const ppc64::implementation* get_ppc64_singleton() {
|
||||
static const ppc64::implementation ppc64_singleton{};
|
||||
return &ppc64_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_PPC64
|
||||
#if SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
const fallback::implementation fallback_singleton{};
|
||||
static const fallback::implementation* get_fallback_singleton() {
|
||||
static const fallback::implementation fallback_singleton{};
|
||||
return &fallback_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
|
||||
/**
|
||||
@@ -2661,25 +2676,26 @@ private:
|
||||
const implementation *set_best() const noexcept;
|
||||
};
|
||||
|
||||
const detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton;
|
||||
|
||||
const std::initializer_list<const implementation *> available_implementation_pointers {
|
||||
static const std::initializer_list<const implementation *>& get_available_implementation_pointers() {
|
||||
static const std::initializer_list<const implementation *> available_implementation_pointers {
|
||||
#if SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
&haswell_singleton,
|
||||
get_haswell_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
&westmere_singleton,
|
||||
get_westmere_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64
|
||||
&arm64_singleton,
|
||||
get_arm64_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_PPC64
|
||||
&ppc64_singleton,
|
||||
get_ppc64_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
&fallback_singleton,
|
||||
get_fallback_singleton(),
|
||||
#endif
|
||||
}; // available_implementation_pointers
|
||||
}; // available_implementation_pointers
|
||||
return available_implementation_pointers;
|
||||
}
|
||||
|
||||
// So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no support
|
||||
class unsupported_implementation final : public implementation {
|
||||
@@ -2707,25 +2723,28 @@ public:
|
||||
unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {}
|
||||
};
|
||||
|
||||
const unsupported_implementation unsupported_singleton{};
|
||||
const unsupported_implementation* get_unsupported_singleton() {
|
||||
static const unsupported_implementation unsupported_singleton{};
|
||||
return &unsupported_singleton;
|
||||
}
|
||||
|
||||
size_t available_implementation_list::size() const noexcept {
|
||||
return internal::available_implementation_pointers.size();
|
||||
return internal::get_available_implementation_pointers().size();
|
||||
}
|
||||
const implementation * const *available_implementation_list::begin() const noexcept {
|
||||
return internal::available_implementation_pointers.begin();
|
||||
return internal::get_available_implementation_pointers().begin();
|
||||
}
|
||||
const implementation * const *available_implementation_list::end() const noexcept {
|
||||
return internal::available_implementation_pointers.end();
|
||||
return internal::get_available_implementation_pointers().end();
|
||||
}
|
||||
const implementation *available_implementation_list::detect_best_supported() const noexcept {
|
||||
// They are prelisted in priority order, so we just go down the list
|
||||
uint32_t supported_instruction_sets = internal::detect_supported_architectures();
|
||||
for (const implementation *impl : internal::available_implementation_pointers) {
|
||||
for (const implementation *impl : internal::get_available_implementation_pointers()) {
|
||||
uint32_t required_instruction_sets = impl->required_instruction_sets();
|
||||
if ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets) { return impl; }
|
||||
}
|
||||
return &unsupported_singleton; // this should never happen?
|
||||
return get_unsupported_singleton(); // this should never happen?
|
||||
}
|
||||
|
||||
const implementation *detect_best_supported_implementation_on_first_use::set_best() const noexcept {
|
||||
@@ -2735,31 +2754,39 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
if (force_implementation_name) {
|
||||
auto force_implementation = available_implementations[force_implementation_name];
|
||||
auto force_implementation = get_available_implementations()[force_implementation_name];
|
||||
if (force_implementation) {
|
||||
return active_implementation = force_implementation;
|
||||
return get_active_implementation() = force_implementation;
|
||||
} else {
|
||||
// Note: abort() and stderr usage within the library is forbidden.
|
||||
return active_implementation = &unsupported_singleton;
|
||||
return get_active_implementation() = get_unsupported_singleton();
|
||||
}
|
||||
}
|
||||
return active_implementation = available_implementations.detect_best_supported();
|
||||
return get_active_implementation() = get_available_implementations().detect_best_supported();
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{};
|
||||
SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton};
|
||||
SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations() {
|
||||
static const internal::available_implementation_list available_implementations{};
|
||||
return available_implementations;
|
||||
}
|
||||
|
||||
SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation() {
|
||||
static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton;
|
||||
static internal::atomic_ptr<const implementation> active_implementation{&detect_best_supported_implementation_on_first_use_singleton};
|
||||
return active_implementation;
|
||||
}
|
||||
|
||||
simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept {
|
||||
return active_implementation->minify(reinterpret_cast<const uint8_t *>(buf), len, reinterpret_cast<uint8_t *>(dst), dst_len);
|
||||
return get_active_implementation()->minify(reinterpret_cast<const uint8_t *>(buf), len, reinterpret_cast<uint8_t *>(dst), dst_len);
|
||||
}
|
||||
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
|
||||
return active_implementation->validate_utf8(buf, len);
|
||||
return get_active_implementation()->validate_utf8(buf, len);
|
||||
}
|
||||
|
||||
const implementation * builtin_implementation() {
|
||||
static const implementation * builtin_impl = available_implementations[STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
|
||||
static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
|
||||
assert(builtin_impl);
|
||||
return builtin_impl;
|
||||
}
|
||||
@@ -3473,11 +3500,11 @@ simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow
|
||||
|
||||
simdjson_really_inline json_block json_scanner::next(const simd::simd8x64<uint8_t>& in) {
|
||||
json_string_block strings = string_scanner.next(in);
|
||||
// identifies the white-space and the structurat characters
|
||||
// identifies the white-space and the structural characters
|
||||
json_character_block characters = json_character_block::classify(in);
|
||||
// The term "scalar" refers to anything except structural characters and white space
|
||||
// (so letters, numbers, quotes).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
//
|
||||
// A terminal quote should either be followed by a structural character (comma, brace, bracket, colon)
|
||||
// or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential
|
||||
@@ -4246,25 +4273,25 @@ public:
|
||||
/**
|
||||
* Log that a value has been found.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the start of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_start_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the end of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_end_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log an error.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_error(const char *error) const noexcept;
|
||||
|
||||
@@ -5574,25 +5601,25 @@ public:
|
||||
/**
|
||||
* Log that a value has been found.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the start of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_start_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the end of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_end_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log an error.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_error(const char *error) const noexcept;
|
||||
|
||||
@@ -6937,11 +6964,11 @@ simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow
|
||||
|
||||
simdjson_really_inline json_block json_scanner::next(const simd::simd8x64<uint8_t>& in) {
|
||||
json_string_block strings = string_scanner.next(in);
|
||||
// identifies the white-space and the structurat characters
|
||||
// identifies the white-space and the structural characters
|
||||
json_character_block characters = json_character_block::classify(in);
|
||||
// The term "scalar" refers to anything except structural characters and white space
|
||||
// (so letters, numbers, quotes).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
//
|
||||
// A terminal quote should either be followed by a structural character (comma, brace, bracket, colon)
|
||||
// or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential
|
||||
@@ -7709,25 +7736,25 @@ public:
|
||||
/**
|
||||
* Log that a value has been found.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the start of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_start_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the end of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_end_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log an error.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_error(const char *error) const noexcept;
|
||||
|
||||
@@ -9063,11 +9090,11 @@ simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow
|
||||
|
||||
simdjson_really_inline json_block json_scanner::next(const simd::simd8x64<uint8_t>& in) {
|
||||
json_string_block strings = string_scanner.next(in);
|
||||
// identifies the white-space and the structurat characters
|
||||
// identifies the white-space and the structural characters
|
||||
json_character_block characters = json_character_block::classify(in);
|
||||
// The term "scalar" refers to anything except structural characters and white space
|
||||
// (so letters, numbers, quotes).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
//
|
||||
// A terminal quote should either be followed by a structural character (comma, brace, bracket, colon)
|
||||
// or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential
|
||||
@@ -9836,25 +9863,25 @@ public:
|
||||
/**
|
||||
* Log that a value has been found.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the start of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_start_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the end of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_end_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log an error.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_error(const char *error) const noexcept;
|
||||
|
||||
@@ -11225,11 +11252,11 @@ simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow
|
||||
|
||||
simdjson_really_inline json_block json_scanner::next(const simd::simd8x64<uint8_t>& in) {
|
||||
json_string_block strings = string_scanner.next(in);
|
||||
// identifies the white-space and the structurat characters
|
||||
// identifies the white-space and the structural characters
|
||||
json_character_block characters = json_character_block::classify(in);
|
||||
// The term "scalar" refers to anything except structural characters and white space
|
||||
// (so letters, numbers, quotes).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
//
|
||||
// A terminal quote should either be followed by a structural character (comma, brace, bracket, colon)
|
||||
// or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential
|
||||
@@ -11997,25 +12024,25 @@ public:
|
||||
/**
|
||||
* Log that a value has been found.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the start of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_start_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the end of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_end_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log an error.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_error(const char *error) const noexcept;
|
||||
|
||||
|
||||
+119
-71
@@ -1,4 +1,4 @@
|
||||
/* auto-generated on 2021-10-27 19:25:23 -0400. Do not edit! */
|
||||
/* auto-generated on 2022-01-31 11:38:54 -0500. Do not edit! */
|
||||
/* begin file include/simdjson.h */
|
||||
#ifndef SIMDJSON_H
|
||||
#define SIMDJSON_H
|
||||
@@ -36,6 +36,34 @@
|
||||
}
|
||||
*/
|
||||
|
||||
/* begin file include/simdjson/simdjson_version.h */
|
||||
// /include/simdjson/simdjson_version.h automatically generated by release.py,
|
||||
// do not change by hand
|
||||
#ifndef SIMDJSON_SIMDJSON_VERSION_H
|
||||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION 1.0.2
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
/**
|
||||
* The major version (MAJOR.minor.revision) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_MAJOR = 1,
|
||||
/**
|
||||
* The minor version (major.MINOR.revision) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_MINOR = 0,
|
||||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_REVISION = 2
|
||||
};
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_SIMDJSON_VERSION_H
|
||||
/* end file include/simdjson/simdjson_version.h */
|
||||
/* begin file include/simdjson/dom.h */
|
||||
#ifndef SIMDJSON_DOM_H
|
||||
#define SIMDJSON_DOM_H
|
||||
@@ -160,10 +188,10 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
|
||||
#endif // SIMDJSON_IS_32BITS
|
||||
|
||||
// this is almost standard?
|
||||
#undef STRINGIFY_IMPLEMENTATION_
|
||||
#undef STRINGIFY
|
||||
#define STRINGIFY_IMPLEMENTATION_(a) #a
|
||||
#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a)
|
||||
#undef SIMDJSON_STRINGIFY_IMPLEMENTATION_
|
||||
#undef SIMDJSON_STRINGIFY
|
||||
#define SIMDJSON_STRINGIFY_IMPLEMENTATION_(a) #a
|
||||
#define SIMDJSON_STRINGIFY(a) SIMDJSON_STRINGIFY_IMPLEMENTATION_(a)
|
||||
|
||||
// Our fast kernels require 64-bit systems.
|
||||
//
|
||||
@@ -187,13 +215,13 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
|
||||
// til 8.0 so SIMDJSON_TARGET_REGION and SIMDJSON_UNTARGET_REGION must be *outside* of a
|
||||
// namespace.
|
||||
#define SIMDJSON_TARGET_REGION(T) \
|
||||
_Pragma(STRINGIFY( \
|
||||
_Pragma(SIMDJSON_STRINGIFY( \
|
||||
clang attribute push(__attribute__((target(T))), apply_to = function)))
|
||||
#define SIMDJSON_UNTARGET_REGION _Pragma("clang attribute pop")
|
||||
#elif defined(__GNUC__)
|
||||
// GCC is easier
|
||||
#define SIMDJSON_TARGET_REGION(T) \
|
||||
_Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T)))
|
||||
_Pragma("GCC push_options") _Pragma(SIMDJSON_STRINGIFY(GCC target(T)))
|
||||
#define SIMDJSON_UNTARGET_REGION _Pragma("GCC pop_options")
|
||||
#endif // clang then gcc
|
||||
|
||||
@@ -227,11 +255,11 @@ use a 64-bit target such as x64, 64-bit ARM or 64-bit PPC.")
|
||||
|
||||
|
||||
#if defined(__clang__)
|
||||
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
|
||||
#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
|
||||
#elif defined(__GNUC__)
|
||||
#define NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
|
||||
#define SIMDJSON_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
|
||||
#else
|
||||
#define NO_SANITIZE_UNDEFINED
|
||||
#define SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
#endif
|
||||
|
||||
#ifdef SIMDJSON_VISUAL_STUDIO
|
||||
@@ -2182,34 +2210,6 @@ SIMDJSON_PUSH_DISABLE_WARNINGS
|
||||
SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
||||
|
||||
// Public API
|
||||
/* begin file include/simdjson/simdjson_version.h */
|
||||
// /include/simdjson/simdjson_version.h automatically generated by release.py,
|
||||
// do not change by hand
|
||||
#ifndef SIMDJSON_SIMDJSON_VERSION_H
|
||||
#define SIMDJSON_SIMDJSON_VERSION_H
|
||||
|
||||
/** The version of simdjson being used (major.minor.revision) */
|
||||
#define SIMDJSON_VERSION 1.0.2
|
||||
|
||||
namespace simdjson {
|
||||
enum {
|
||||
/**
|
||||
* The major version (MAJOR.minor.revision) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_MAJOR = 1,
|
||||
/**
|
||||
* The minor version (major.MINOR.revision) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_MINOR = 0,
|
||||
/**
|
||||
* The revision (major.minor.REVISION) of simdjson being used.
|
||||
*/
|
||||
SIMDJSON_VERSION_REVISION = 2
|
||||
};
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_SIMDJSON_VERSION_H
|
||||
/* end file include/simdjson/simdjson_version.h */
|
||||
/* begin file include/simdjson/error.h */
|
||||
#ifndef SIMDJSON_ERROR_H
|
||||
#define SIMDJSON_ERROR_H
|
||||
@@ -2219,7 +2219,12 @@ enum {
|
||||
namespace simdjson {
|
||||
|
||||
/**
|
||||
* All possible errors returned by simdjson.
|
||||
* All possible errors returned by simdjson. These error codes are subject to change
|
||||
* and not all simdjson kernel returns the same error code given the same input: it is not
|
||||
* well defined which error a given input should produce.
|
||||
*
|
||||
* Only SUCCESS evaluates to false as a Boolean. All other error codes will evaluate
|
||||
* to true as a Boolean.
|
||||
*/
|
||||
enum error_code {
|
||||
SUCCESS = 0, ///< No error
|
||||
@@ -3247,7 +3252,7 @@ public:
|
||||
/**
|
||||
* The name of this implementation.
|
||||
*
|
||||
* const implementation *impl = simdjson::active_implementation;
|
||||
* const implementation *impl = simdjson::get_active_implementation();
|
||||
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
|
||||
*
|
||||
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
|
||||
@@ -3257,7 +3262,7 @@ public:
|
||||
/**
|
||||
* The description of this implementation.
|
||||
*
|
||||
* const implementation *impl = simdjson::active_implementation;
|
||||
* const implementation *impl = simdjson::get_active_implementation();
|
||||
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
|
||||
*
|
||||
* @return the name of the implementation, e.g. "haswell", "westmere", "arm64"
|
||||
@@ -3286,7 +3291,7 @@ public:
|
||||
/**
|
||||
* @private For internal implementation use
|
||||
*
|
||||
* const implementation *impl = simdjson::active_implementation;
|
||||
* const implementation *impl = simdjson::get_active_implementation();
|
||||
* cout << "simdjson is optimized for " << impl->name() << "(" << impl->description() << ")" << endl;
|
||||
*
|
||||
* @param capacity The largest document that will be passed to the parser.
|
||||
@@ -3380,10 +3385,10 @@ public:
|
||||
*
|
||||
* Case sensitive.
|
||||
*
|
||||
* const implementation *impl = simdjson::available_implementations["westmere"];
|
||||
* const implementation *impl = simdjson::get_available_implementations()["westmere"];
|
||||
* if (!impl) { exit(1); }
|
||||
* if (!imp->supported_by_runtime_system()) { exit(1); }
|
||||
* simdjson::active_implementation = impl;
|
||||
* simdjson::get_active_implementation() = impl;
|
||||
*
|
||||
* @param name the implementation to find, e.g. "westmere", "haswell", "arm64"
|
||||
* @return the implementation, or nullptr if the parse failed.
|
||||
@@ -3401,7 +3406,7 @@ public:
|
||||
* This is used to initialize the implementation on startup.
|
||||
*
|
||||
* const implementation *impl = simdjson::available_implementation::detect_best_supported();
|
||||
* simdjson::active_implementation = impl;
|
||||
* simdjson::get_active_implementation() = impl;
|
||||
*
|
||||
* @return the most advanced supported implementation for the current host, or an
|
||||
* implementation that returns UNSUPPORTED_ARCHITECTURE if there is no supported
|
||||
@@ -3433,14 +3438,14 @@ private:
|
||||
/**
|
||||
* The list of available implementations compiled into simdjson.
|
||||
*/
|
||||
extern SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations;
|
||||
extern SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations();
|
||||
|
||||
/**
|
||||
* The active implementation.
|
||||
*
|
||||
* Automatically initialized on first use to the most advanced implementation supported by this hardware.
|
||||
*/
|
||||
extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation;
|
||||
extern SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation();
|
||||
|
||||
} // namespace simdjson
|
||||
|
||||
@@ -3686,7 +3691,7 @@ inline padded_string::padded_string(size_t length) noexcept
|
||||
}
|
||||
inline padded_string::padded_string(const char *data, size_t length) noexcept
|
||||
: viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) {
|
||||
if ((data != nullptr) and (data_ptr != nullptr)) {
|
||||
if ((data != nullptr) && (data_ptr != nullptr)) {
|
||||
std::memcpy(data_ptr, data, length);
|
||||
}
|
||||
}
|
||||
@@ -8672,7 +8677,7 @@ inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept {
|
||||
if (implementation) {
|
||||
err = implementation->allocate(capacity, max_depth);
|
||||
} else {
|
||||
err = simdjson::active_implementation->create_dom_parser_implementation(capacity, max_depth, implementation);
|
||||
err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation);
|
||||
}
|
||||
if (err) { return err; }
|
||||
return SUCCESS;
|
||||
@@ -9420,9 +9425,13 @@ extern SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256];
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
#define SIMDJSON_IMPLEMENTATION_HASWELL (SIMDJSON_IS_X86_64)
|
||||
#endif
|
||||
#ifdef _MSC_VER
|
||||
// To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see
|
||||
// https://github.com/simdjson/simdjson/issues/1247
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__))
|
||||
#else
|
||||
#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__))
|
||||
#endif
|
||||
|
||||
// Default Westmere to on if this is x86-64, unless we'll always select Haswell.
|
||||
#ifndef SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
@@ -9598,7 +9607,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
unsigned long ret;
|
||||
@@ -9663,7 +9672,7 @@ simdjson_really_inline uint64_t reverse_bits(uint64_t input_num) {
|
||||
* greating or equal to 63 in which case we trigger undefined behavior, but the output
|
||||
* of such undefined behavior is never used.
|
||||
**/
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) {
|
||||
return rev_bits ^ (uint64_t(0x8000000000000000) >> leading_zeroes);
|
||||
}
|
||||
@@ -9860,7 +9869,7 @@ simdjson_really_inline int8x16_t make_int8x16_t(int8_t x1, int8_t x2, int8_t x
|
||||
simdjson_really_inline simd8<T>& operator&=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast & other; return *this_cast; }
|
||||
simdjson_really_inline simd8<T>& operator^=(const simd8<T> other) { auto this_cast = static_cast<simd8<T>*>(this); *this_cast = *this_cast ^ other; return *this_cast; }
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const { return vceqq_u8(*this, other); }
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return vceqq_u8(lhs, rhs); }
|
||||
|
||||
template<int N=1>
|
||||
simdjson_really_inline simd8<T> prev(const simd8<T> prev_chunk) const {
|
||||
@@ -11021,7 +11030,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
||||
}
|
||||
|
||||
template<typename I>
|
||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
||||
if (digit > 9) {
|
||||
@@ -12854,7 +12863,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
||||
}
|
||||
|
||||
template<typename I>
|
||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
||||
if (digit > 9) {
|
||||
@@ -13961,7 +13970,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
return (int)_tzcnt_u64(input_num);
|
||||
@@ -14087,7 +14096,7 @@ namespace simd {
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m256i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const { return _mm256_cmpeq_epi8(*this, other); }
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return _mm256_cmpeq_epi8(lhs, rhs); }
|
||||
|
||||
static const int SIZE = sizeof(base<T>::value);
|
||||
|
||||
@@ -15172,7 +15181,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
||||
}
|
||||
|
||||
template<typename I>
|
||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
||||
if (digit > 9) {
|
||||
@@ -16234,7 +16243,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
unsigned long ret;
|
||||
@@ -16408,8 +16417,8 @@ struct base8 : base<simd8<T>> {
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const {
|
||||
return (__m128i)vec_cmpeq(this->value, (__m128i)other);
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) {
|
||||
return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs);
|
||||
}
|
||||
|
||||
static const int SIZE = sizeof(base<simd8<T>>::value);
|
||||
@@ -17200,7 +17209,11 @@ simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_str
|
||||
#ifndef SIMDJSON_PPC64_NUMBERPARSING_H
|
||||
#define SIMDJSON_PPC64_NUMBERPARSING_H
|
||||
|
||||
#if defined(__linux__)
|
||||
#include <byteswap.h>
|
||||
#elif defined(__FreeBSD__)
|
||||
#include <sys/endian.h>
|
||||
#endif
|
||||
|
||||
namespace simdjson {
|
||||
namespace ppc64 {
|
||||
@@ -17213,7 +17226,11 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||
uint64_t val;
|
||||
std::memcpy(&val, chars, sizeof(uint64_t));
|
||||
#ifdef __BIG_ENDIAN__
|
||||
#if defined(__linux__)
|
||||
val = bswap_64(val);
|
||||
#elif defined(__FreeBSD__)
|
||||
val = bswap64(val);
|
||||
#endif
|
||||
#endif
|
||||
val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8;
|
||||
val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16;
|
||||
@@ -17589,7 +17606,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
||||
}
|
||||
|
||||
template<typename I>
|
||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
||||
if (digit > 9) {
|
||||
@@ -18672,7 +18689,7 @@ namespace {
|
||||
// We sometimes call trailing_zero on inputs that are zero,
|
||||
// but the algorithms do not end up using the returned value.
|
||||
// Sadly, sanitizers are not smart enough to figure it out.
|
||||
NO_SANITIZE_UNDEFINED
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED
|
||||
simdjson_really_inline int trailing_zeroes(uint64_t input_num) {
|
||||
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||
unsigned long ret;
|
||||
@@ -18806,7 +18823,7 @@ namespace simd {
|
||||
simdjson_really_inline base8() : base<simd8<T>>() {}
|
||||
simdjson_really_inline base8(const __m128i _value) : base<simd8<T>>(_value) {}
|
||||
|
||||
simdjson_really_inline Mask operator==(const simd8<T> other) const { return _mm_cmpeq_epi8(*this, other); }
|
||||
friend simdjson_really_inline Mask operator==(const simd8<T> lhs, const simd8<T> rhs) { return _mm_cmpeq_epi8(lhs, rhs); }
|
||||
|
||||
static const int SIZE = sizeof(base<simd8<T>>::value);
|
||||
|
||||
@@ -19864,7 +19881,7 @@ error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
||||
}
|
||||
|
||||
template<typename I>
|
||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
||||
if (digit > 9) {
|
||||
@@ -21048,7 +21065,6 @@ protected:
|
||||
uint64_t unsigned_integer;
|
||||
} payload{0};
|
||||
number_type type{number_type::signed_integer};
|
||||
friend class value_iterator;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -22512,7 +22528,7 @@ public:
|
||||
|
||||
/**
|
||||
* Get the value at the given index. This function has linear-time complexity.
|
||||
* This function should only be called once as the array iterator is not reset between each call.
|
||||
* This function should only be called once on an array instance since the array iterator is not reset between each call.
|
||||
*
|
||||
* @return The value at the given index, or:
|
||||
* - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
|
||||
@@ -22694,6 +22710,8 @@ public:
|
||||
*
|
||||
* The string is guaranteed to be valid UTF-8.
|
||||
*
|
||||
* Important: Calling get_string() twice on the same document is an error.
|
||||
*
|
||||
* @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
|
||||
* time it parses a document or when it is destroyed.
|
||||
* @returns INCORRECT_TYPE if the JSON value is not a string.
|
||||
@@ -22867,7 +22885,7 @@ public:
|
||||
simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
|
||||
/**
|
||||
* Get the value at the given index in the array. This function has linear-time complexity.
|
||||
* This function should only be called once as the array iterator is not reset between each call.
|
||||
* This function should only be called once on an array instance since the array iterator is not reset between each call.
|
||||
*
|
||||
* @return The value at the given index, or:
|
||||
* - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
|
||||
@@ -22911,6 +22929,10 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to
|
||||
* a key a single time. Doing object["mykey"].to_string()and then again object["mykey"].to_string()
|
||||
* is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -22941,6 +22963,10 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to a key
|
||||
* a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string()
|
||||
* is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -23113,7 +23139,7 @@ public:
|
||||
simdjson_really_inline simdjson_result<value> at_pointer(std::string_view json_pointer) noexcept;
|
||||
/**
|
||||
* Consumes the document and returns a string_view instance corresponding to the
|
||||
* document as represented in JSON. It points inside the original byte array containg
|
||||
* document as represented in JSON. It points inside the original byte array containing
|
||||
* the JSON document.
|
||||
*/
|
||||
simdjson_really_inline simdjson_result<std::string_view> raw_json() noexcept;
|
||||
@@ -23461,6 +23487,9 @@ public:
|
||||
*
|
||||
* Equivalent to get<std::string_view>().
|
||||
*
|
||||
* Important: a value should be consumed once. Calling get_string() twice on the same value
|
||||
* is an error.
|
||||
*
|
||||
* @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next
|
||||
* time it parses a document or when it is destroyed.
|
||||
* @returns INCORRECT_TYPE if the JSON value is not a string.
|
||||
@@ -23601,7 +23630,7 @@ public:
|
||||
simdjson_really_inline simdjson_result<size_t> count_fields() & noexcept;
|
||||
/**
|
||||
* Get the value at the given index in the array. This function has linear-time complexity.
|
||||
* This function should only be called once as the array iterator is not reset between each call.
|
||||
* This function should only be called once on an array instance since the array iterator is not reset between each call.
|
||||
*
|
||||
* @return The value at the given index, or:
|
||||
* - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length
|
||||
@@ -24112,6 +24141,10 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to a
|
||||
* key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string()
|
||||
* is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -24145,6 +24178,9 @@ public:
|
||||
* the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an
|
||||
* OUT_OF_ORDER_ITERATION error is generated.
|
||||
*
|
||||
* You are expected to access keys only once. You should access the value corresponding to a key
|
||||
* a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error.
|
||||
*
|
||||
* @param key The key to look up.
|
||||
* @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
@@ -24468,7 +24504,7 @@ public:
|
||||
* The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8).
|
||||
*
|
||||
* The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
|
||||
* Setting batch_size to excessively large or excesively small values may impact negatively the
|
||||
* Setting batch_size to excessively large or excessively small values may impact negatively the
|
||||
* performance.
|
||||
*
|
||||
* ### REQUIRED: Buffer Padding
|
||||
@@ -24930,6 +24966,14 @@ inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON
|
||||
inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::array> x);
|
||||
} // namespace simdjson
|
||||
|
||||
/**
|
||||
* We want to support argument-dependent lookup (ADL).
|
||||
* Hence we should define operator<< in the namespace
|
||||
* where the argument (here value, object, etc.) resides.
|
||||
* Credit: @madhur4127
|
||||
* See https://github.com/simdjson/simdjson/issues/1768
|
||||
*/
|
||||
namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand {
|
||||
|
||||
/**
|
||||
* Print JSON to an output stream.
|
||||
@@ -24979,6 +25023,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_BUILTIN_IM
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result<simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::object> x);
|
||||
#endif
|
||||
}}} // namespace simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand
|
||||
/* end file include/simdjson/generic/ondemand/serialization.h */
|
||||
/* end file include/simdjson/generic/ondemand.h */
|
||||
|
||||
@@ -28108,6 +28153,7 @@ template<> simdjson_really_inline simdjson_result<array> value::get() noexcept {
|
||||
template<> simdjson_really_inline simdjson_result<object> value::get() noexcept { return get_object(); }
|
||||
template<> simdjson_really_inline simdjson_result<raw_json_string> value::get() noexcept { return get_raw_json_string(); }
|
||||
template<> simdjson_really_inline simdjson_result<std::string_view> value::get() noexcept { return get_string(); }
|
||||
template<> simdjson_really_inline simdjson_result<number> value::get() noexcept { return get_number(); }
|
||||
template<> simdjson_really_inline simdjson_result<double> value::get() noexcept { return get_double(); }
|
||||
template<> simdjson_really_inline simdjson_result<uint64_t> value::get() noexcept { return get_uint64(); }
|
||||
template<> simdjson_really_inline simdjson_result<int64_t> value::get() noexcept { return get_int64(); }
|
||||
@@ -28787,7 +28833,7 @@ simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t n
|
||||
SIMDJSON_TRY( implementation->set_capacity(new_capacity) );
|
||||
SIMDJSON_TRY( implementation->set_max_depth(new_max_depth) );
|
||||
} else {
|
||||
SIMDJSON_TRY( simdjson::active_implementation->create_dom_parser_implementation(new_capacity, new_max_depth, implementation) );
|
||||
SIMDJSON_TRY( simdjson::get_active_implementation()->create_dom_parser_implementation(new_capacity, new_max_depth, implementation) );
|
||||
}
|
||||
_capacity = new_capacity;
|
||||
_max_depth = new_max_depth;
|
||||
@@ -29407,6 +29453,7 @@ inline simdjson_result<std::string_view> to_json_string(simdjson_result<SIMDJSON
|
||||
}
|
||||
} // namespace simdjson
|
||||
|
||||
namespace simdjson { namespace SIMDJSON_BUILTIN_IMPLEMENTATION { namespace ondemand {
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand::value x) {
|
||||
@@ -29524,6 +29571,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::SIMDJSON_BUILTIN_IM
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}}} // namespace simdjson::SIMDJSON_BUILTIN_IMPLEMENTATION::ondemand
|
||||
/* end file include/simdjson/generic/ondemand/serialization-inl.h */
|
||||
/* end file include/simdjson/generic/ondemand-inl.h */
|
||||
|
||||
|
||||
@@ -124,11 +124,11 @@ simdjson_really_inline uint64_t follows(const uint64_t match, uint64_t &overflow
|
||||
|
||||
simdjson_really_inline json_block json_scanner::next(const simd::simd8x64<uint8_t>& in) {
|
||||
json_string_block strings = string_scanner.next(in);
|
||||
// identifies the white-space and the structurat characters
|
||||
// identifies the white-space and the structural characters
|
||||
json_character_block characters = json_character_block::classify(in);
|
||||
// The term "scalar" refers to anything except structural characters and white space
|
||||
// (so letters, numbers, quotes).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
// We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers).
|
||||
//
|
||||
// A terminal quote should either be followed by a structural character (comma, brace, bracket, colon)
|
||||
// or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential
|
||||
|
||||
@@ -25,6 +25,12 @@ public:
|
||||
// base_ptr[base] incrementing base as we go
|
||||
// will potentially store extra values beyond end of valid bits, so base_ptr
|
||||
// needs to be large enough to handle this
|
||||
//
|
||||
// If the kernel sets SIMDJSON_CUSTOM_BIT_INDEXER, then it will provide its own
|
||||
// version of the code.
|
||||
#ifdef SIMDJSON_CUSTOM_BIT_INDEXER
|
||||
simdjson_really_inline void write(uint32_t idx, uint64_t bits);
|
||||
#else
|
||||
simdjson_really_inline void write(uint32_t idx, uint64_t bits) {
|
||||
// In some instances, the next branch is expensive because it is mispredicted.
|
||||
// Unfortunately, in other cases,
|
||||
@@ -117,6 +123,8 @@ public:
|
||||
this->tail += cnt;
|
||||
#endif
|
||||
}
|
||||
#endif // SIMDJSON_CUSTOM_BIT_INDEXER
|
||||
|
||||
};
|
||||
|
||||
class json_structural_indexer {
|
||||
|
||||
@@ -111,12 +111,25 @@ using namespace simd;
|
||||
simdjson_really_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
|
||||
// If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
|
||||
// ... 1111____ 111_____ 11______
|
||||
#if SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
static const uint8_t max_array[64] = {
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1
|
||||
};
|
||||
#else
|
||||
static const uint8_t max_array[32] = {
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 255, 255, 255,
|
||||
255, 255, 255, 255, 255, 0b11110000u-1, 0b11100000u-1, 0b11000000u-1
|
||||
};
|
||||
#endif
|
||||
const simd8<uint8_t> max_value(&max_array[sizeof(max_array)-sizeof(simd8<uint8_t>)]);
|
||||
return input.gt_bits(max_value);
|
||||
}
|
||||
@@ -154,9 +167,13 @@ using namespace simd;
|
||||
this->error |= this->prev_incomplete;
|
||||
} else {
|
||||
// you might think that a for-loop would work, but under Visual Studio, it is not good enough.
|
||||
static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 2) || (simd8x64<uint8_t>::NUM_CHUNKS == 4),
|
||||
"We support either two or four chunks per 64-byte block.");
|
||||
if(simd8x64<uint8_t>::NUM_CHUNKS == 2) {
|
||||
static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 1)
|
||||
||(simd8x64<uint8_t>::NUM_CHUNKS == 2)
|
||||
|| (simd8x64<uint8_t>::NUM_CHUNKS == 4),
|
||||
"We support one, two or four chunks per 64-byte block.");
|
||||
if(simd8x64<uint8_t>::NUM_CHUNKS == 1) {
|
||||
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
|
||||
} if(simd8x64<uint8_t>::NUM_CHUNKS == 2) {
|
||||
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
|
||||
this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
|
||||
} else if(simd8x64<uint8_t>::NUM_CHUNKS == 4) {
|
||||
|
||||
@@ -80,25 +80,25 @@ public:
|
||||
/**
|
||||
* Log that a value has been found.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the start of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_start_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log the end of a multipart value.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_end_value(const char *type) const noexcept;
|
||||
/**
|
||||
* Log an error.
|
||||
*
|
||||
* Set ENABLE_LOGGING=true in logger.h to see logging.
|
||||
* Set LOG_ENABLED=true in logger.h to see logging.
|
||||
*/
|
||||
simdjson_really_inline void log_error(const char *error) const noexcept;
|
||||
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
#include "simdjson/icelake/begin.h"
|
||||
|
||||
//
|
||||
// Stage 1
|
||||
//
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
|
||||
using namespace simd;
|
||||
|
||||
struct json_character_block {
|
||||
static simdjson_really_inline json_character_block classify(const simd::simd8x64<uint8_t>& in);
|
||||
// ASCII white-space ('\r','\n','\t',' ')
|
||||
simdjson_really_inline uint64_t whitespace() const noexcept;
|
||||
// non-quote structural characters (comma, colon, braces, brackets)
|
||||
simdjson_really_inline uint64_t op() const noexcept;
|
||||
// neither a structural character nor a white-space, so letters, numbers and quotes
|
||||
simdjson_really_inline uint64_t scalar() const noexcept;
|
||||
|
||||
uint64_t _whitespace; // ASCII white-space ('\r','\n','\t',' ')
|
||||
uint64_t _op; // structural characters (comma, colon, braces, brackets but not quotes)
|
||||
};
|
||||
|
||||
simdjson_really_inline uint64_t json_character_block::whitespace() const noexcept { return _whitespace; }
|
||||
simdjson_really_inline uint64_t json_character_block::op() const noexcept { return _op; }
|
||||
simdjson_really_inline uint64_t json_character_block::scalar() const noexcept { return ~(op() | whitespace()); }
|
||||
|
||||
// This identifies structural characters (comma, colon, braces, brackets),
|
||||
// and ASCII white-space ('\r','\n','\t',' ').
|
||||
simdjson_really_inline json_character_block json_character_block::classify(const simd::simd8x64<uint8_t>& in) {
|
||||
// These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why
|
||||
// we can't use the generic lookup_16.
|
||||
const auto whitespace_table = simd8<uint8_t>::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100);
|
||||
|
||||
// The 6 operators (:,[]{}) have these values:
|
||||
//
|
||||
// , 2C
|
||||
// : 3A
|
||||
// [ 5B
|
||||
// { 7B
|
||||
// ] 5D
|
||||
// } 7D
|
||||
//
|
||||
// If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique.
|
||||
// We exploit this, using a simd 4-bit lookup to tell us which character match against, and then
|
||||
// match it (against | 0x20).
|
||||
//
|
||||
// To prevent recognizing other characters, everything else gets compared with 0, which cannot
|
||||
// match due to the | 0x20.
|
||||
//
|
||||
// NOTE: Due to the | 0x20, this ALSO treats <FF> and <SUB> (control characters 0C and 1A) like ,
|
||||
// and :. This gets caught in stage 2, which checks the actual character to ensure the right
|
||||
// operators are in the right places.
|
||||
const auto op_table = simd8<uint8_t>::repeat_16(
|
||||
0, 0, 0, 0,
|
||||
0, 0, 0, 0,
|
||||
0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B
|
||||
',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D
|
||||
);
|
||||
|
||||
// We compute whitespace and op separately. If later code only uses one or the
|
||||
// other, given the fact that all functions are aggressively inlined, we can
|
||||
// hope that useless computations will be omitted. This is namely case when
|
||||
// minifying (we only need whitespace).
|
||||
|
||||
const uint64_t whitespace = in.eq({
|
||||
_mm512_shuffle_epi8(whitespace_table, in.chunks[0])
|
||||
});
|
||||
// Turn [ and ] into { and }
|
||||
const simd8x64<uint8_t> curlified{
|
||||
in.chunks[0] | 0x20
|
||||
};
|
||||
const uint64_t op = curlified.eq({
|
||||
_mm512_shuffle_epi8(op_table, in.chunks[0])
|
||||
});
|
||||
|
||||
return { whitespace, op };
|
||||
}
|
||||
|
||||
simdjson_really_inline bool is_ascii(const simd8x64<uint8_t>& input) {
|
||||
return input.reduce_or().is_ascii();
|
||||
}
|
||||
|
||||
simdjson_unused simdjson_really_inline simd8<bool> must_be_continuation(const simd8<uint8_t> prev1, const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
|
||||
simd8<uint8_t> is_second_byte = prev1.saturating_sub(0b11000000u-1); // Only 11______ will be > 0
|
||||
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
|
||||
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
|
||||
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
|
||||
return simd8<int8_t>(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0);
|
||||
}
|
||||
|
||||
simdjson_really_inline simd8<bool> must_be_2_3_continuation(const simd8<uint8_t> prev2, const simd8<uint8_t> prev3) {
|
||||
simd8<uint8_t> is_third_byte = prev2.saturating_sub(0b11100000u-1); // Only 111_____ will be > 0
|
||||
simd8<uint8_t> is_fourth_byte = prev3.saturating_sub(0b11110000u-1); // Only 1111____ will be > 0
|
||||
// Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine.
|
||||
return simd8<int8_t>(is_third_byte | is_fourth_byte) > int8_t(0);
|
||||
}
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#include "icelake/utf8_lookup4_algorithm.h"
|
||||
// defining SIMDJSON_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write
|
||||
#define SIMDJSON_CUSTOM_BIT_INDEXER
|
||||
#include "generic/stage1/json_structural_indexer.h"
|
||||
// We must not forget to undefine it now:
|
||||
#undef SIMDJSON_CUSTOM_BIT_INDEXER
|
||||
|
||||
/**
|
||||
* We provide a custom version of bit_indexer::write using
|
||||
* naked intrinsics.
|
||||
* TODO: make this code more elegant.
|
||||
*/
|
||||
namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace { namespace stage1 {
|
||||
simdjson_really_inline void bit_indexer::write(uint32_t idx, uint64_t bits) {
|
||||
// In some instances, the next branch is expensive because it is mispredicted.
|
||||
// Unfortunately, in other cases,
|
||||
// it helps tremendously.
|
||||
if (bits == 0) { return; }
|
||||
|
||||
const __m512i indexes = _mm512_maskz_compress_epi8(bits, _mm512_set_epi32(
|
||||
0x3f3e3d3c, 0x3b3a3938, 0x37363534, 0x33323130,
|
||||
0x2f2e2d2c, 0x2b2a2928, 0x27262524, 0x23222120,
|
||||
0x1f1e1d1c, 0x1b1a1918, 0x17161514, 0x13121110,
|
||||
0x0f0e0d0c, 0x0b0a0908, 0x07060504, 0x03020100
|
||||
));
|
||||
const __m512i start_index = _mm512_set1_epi32(idx);
|
||||
|
||||
const auto count = count_ones(bits);
|
||||
__m512i t0 = _mm512_cvtepu8_epi32(_mm512_castsi512_si128(indexes));
|
||||
_mm512_storeu_si512(this->tail, _mm512_add_epi32(t0, start_index));
|
||||
|
||||
if(count > 16) {
|
||||
const __m512i t1 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 1));
|
||||
_mm512_storeu_si512(this->tail + 16, _mm512_add_epi32(t1, start_index));
|
||||
if(count > 32) {
|
||||
const __m512i t2 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 2));
|
||||
_mm512_storeu_si512(this->tail + 32, _mm512_add_epi32(t2, start_index));
|
||||
if(count > 48) {
|
||||
const __m512i t3 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 3));
|
||||
_mm512_storeu_si512(this->tail + 48, _mm512_add_epi32(t3, start_index));
|
||||
}
|
||||
}
|
||||
}
|
||||
this->tail += count;
|
||||
}
|
||||
}}}}
|
||||
|
||||
#include "generic/stage1/utf8_validator.h"
|
||||
|
||||
//
|
||||
// Stage 2
|
||||
//
|
||||
#include "generic/stage2/tape_builder.h"
|
||||
|
||||
//
|
||||
// Implementation-specific overrides
|
||||
//
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
namespace stage1 {
|
||||
|
||||
simdjson_really_inline uint64_t json_string_scanner::find_escaped(uint64_t backslash) {
|
||||
if (!backslash) { uint64_t escaped = prev_escaped; prev_escaped = 0; return escaped; }
|
||||
return find_escaped_branchless(backslash);
|
||||
}
|
||||
|
||||
} // namespace stage1
|
||||
} // unnamed namespace
|
||||
|
||||
simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_t len, uint8_t *dst, size_t &dst_len) const noexcept {
|
||||
return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len);
|
||||
}
|
||||
|
||||
simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept {
|
||||
this->buf = _buf;
|
||||
this->len = _len;
|
||||
return icelake::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming);
|
||||
}
|
||||
|
||||
simdjson_warn_unused bool implementation::validate_utf8(const char *buf, size_t len) const noexcept {
|
||||
return icelake::stage1::generic_validate_utf8(buf,len);
|
||||
}
|
||||
|
||||
simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document &_doc) noexcept {
|
||||
return stage2::tape_builder::parse_document<false>(*this, _doc);
|
||||
}
|
||||
|
||||
simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document &_doc) noexcept {
|
||||
return stage2::tape_builder::parse_document<true>(*this, _doc);
|
||||
}
|
||||
|
||||
simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t *_buf, size_t _len, dom::document &_doc) noexcept {
|
||||
auto error = stage1(_buf, _len, stage1_mode::regular);
|
||||
if (error) { return error; }
|
||||
return stage2(_doc);
|
||||
}
|
||||
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#include "simdjson/icelake/end.h"
|
||||
@@ -0,0 +1,24 @@
|
||||
#include "simdjson/icelake/begin.h"
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
|
||||
simdjson_warn_unused error_code implementation::create_dom_parser_implementation(
|
||||
size_t capacity,
|
||||
size_t max_depth,
|
||||
std::unique_ptr<internal::dom_parser_implementation>& dst
|
||||
) const noexcept {
|
||||
dst.reset( new (std::nothrow) dom_parser_implementation() );
|
||||
if (!dst) { return MEMALLOC; }
|
||||
if (auto err = dst->set_capacity(capacity))
|
||||
return err;
|
||||
if (auto err = dst->set_max_depth(max_depth))
|
||||
return err;
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#include "simdjson/icelake/end.h"
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace {
|
||||
namespace utf8_validation {
|
||||
|
||||
simdjson_really_inline __m512i check_special_cases(__m512i input, const __m512i prev1) {
|
||||
__m512i mask1 = _mm512_setr_epi64(
|
||||
0x0202020202020202,
|
||||
0x4915012180808080,
|
||||
0x0202020202020202,
|
||||
0x4915012180808080,
|
||||
0x0202020202020202,
|
||||
0x4915012180808080,
|
||||
0x0202020202020202,
|
||||
0x4915012180808080);
|
||||
|
||||
const __m512i v_0f = _mm512_set1_epi8(0x0f);
|
||||
__m512i index1 = _mm512_and_si512(_mm512_srli_epi16(prev1, 4), v_0f);
|
||||
|
||||
__m512i byte_1_high = _mm512_shuffle_epi8(mask1, index1);
|
||||
__m512i mask2 = _mm512_setr_epi64(
|
||||
0xcbcbcb8b8383a3e7,
|
||||
0xcbcbdbcbcbcbcbcb,
|
||||
0xcbcbcb8b8383a3e7,
|
||||
0xcbcbdbcbcbcbcbcb,
|
||||
0xcbcbcb8b8383a3e7,
|
||||
0xcbcbdbcbcbcbcbcb,
|
||||
0xcbcbcb8b8383a3e7,
|
||||
0xcbcbdbcbcbcbcbcb);
|
||||
__m512i index2 = _mm512_and_si512(prev1, v_0f);
|
||||
|
||||
__m512i byte_1_low = _mm512_shuffle_epi8(mask2, index2);
|
||||
__m512i mask3 = _mm512_setr_epi64(
|
||||
0x101010101010101,
|
||||
0x1010101babaaee6,
|
||||
0x101010101010101,
|
||||
0x1010101babaaee6,
|
||||
0x101010101010101,
|
||||
0x1010101babaaee6,
|
||||
0x101010101010101,
|
||||
0x1010101babaaee6
|
||||
);
|
||||
__m512i index3 = _mm512_and_si512(_mm512_srli_epi16(input, 4), v_0f);
|
||||
__m512i byte_2_high = _mm512_shuffle_epi8(mask3, index3);
|
||||
return _mm512_ternarylogic_epi64(byte_1_high, byte_1_low, byte_2_high, 128);
|
||||
}
|
||||
|
||||
simdjson_really_inline __m512i check_multibyte_lengths(const __m512i prev2,
|
||||
const __m512i prev3, const __m512i sc) {
|
||||
|
||||
__m512i is_third_byte = _mm512_subs_epu8(prev2, _mm512_set1_epi8(0b11100000u-1)); // Only 111_____ will be > 0
|
||||
__m512i is_fourth_byte = _mm512_subs_epu8(prev3, _mm512_set1_epi8(0b11110000u-1)); // Only 1111____ will be > 0
|
||||
__m512i is_third_or_fourth_byte = _mm512_or_si512(is_third_byte, is_fourth_byte);
|
||||
const __m512i v_7f = _mm512_set1_epi8(char(0x7f));
|
||||
is_third_or_fourth_byte = _mm512_adds_epu8(v_7f, is_third_or_fourth_byte);
|
||||
// We want to compute (is_third_or_fourth_byte AND v80) XOR sc.
|
||||
const __m512i v_80 = _mm512_set1_epi8(char(0x80));
|
||||
return _mm512_ternarylogic_epi32(is_third_or_fourth_byte, v_80, sc, 0b1101010);
|
||||
// We could also do it the long way:
|
||||
//
|
||||
//__m512i is_third_or_fourth_byte_mask = _mm512_and_si512(is_third_or_fourth_byte, v_80);
|
||||
//return _mm512_xor_si512(is_third_or_fourth_byte_mask, sc);
|
||||
}
|
||||
//
|
||||
// Return nonzero if there are incomplete multibyte characters at the end of the block:
|
||||
// e.g. if there is a 4-byte character, but it's 3 bytes from the end.
|
||||
//
|
||||
simdjson_really_inline __m512i is_incomplete(const __m512i input) {
|
||||
// If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
|
||||
// ... 1111____ 111_____ 11______
|
||||
const __m512i max_value = _mm512_setr_epi64(
|
||||
0xffffffffffffffff,
|
||||
0xffffffffffffffff,
|
||||
0xffffffffffffffff,
|
||||
0xffffffffffffffff,
|
||||
0xffffffffffffffff,
|
||||
0xffffffffffffffff,
|
||||
0xffffffffffffffff,
|
||||
0xbfdfefffffffffff);
|
||||
return _mm512_subs_epu8(input, max_value);
|
||||
}
|
||||
|
||||
struct utf8_checker {
|
||||
// If this is nonzero, there has been a UTF-8 error.
|
||||
__m512i error{};
|
||||
|
||||
// The last input we received
|
||||
__m512i prev_input_block{};
|
||||
// Whether the last input we received was incomplete (used for ASCII fast path)
|
||||
__m512i prev_incomplete{};
|
||||
|
||||
//
|
||||
// Check whether the current bytes are valid UTF-8.
|
||||
//
|
||||
simdjson_really_inline void check_utf8_bytes(const __m512i input, const __m512i prev_input) {
|
||||
// Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
|
||||
// (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
|
||||
const __m512i movemask = _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6);
|
||||
const __m512i rotated = _mm512_permutex2var_epi64(prev_input, movemask, input);
|
||||
__m512i prev1 = _mm512_alignr_epi8(input, rotated, 16-1);
|
||||
__m512i prev2 = _mm512_alignr_epi8(input, rotated, 16-2);
|
||||
__m512i prev3 = _mm512_alignr_epi8(input, rotated, 16-3);
|
||||
__m512i sc = check_special_cases(input, prev1);
|
||||
this->error = _mm512_or_si512(check_multibyte_lengths(prev2, prev3, sc), this->error);
|
||||
}
|
||||
|
||||
// The only problem that can happen at EOF is that a multibyte character is too short
|
||||
// or a byte value too large in the last bytes: check_special_cases only checks for bytes
|
||||
// too large in the first of two bytes.
|
||||
simdjson_really_inline void check_eof() {
|
||||
// If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't
|
||||
// possibly finish them.
|
||||
this->error = _mm512_or_si512(this->error, this->prev_incomplete);
|
||||
}
|
||||
|
||||
// returns true if ASCII.
|
||||
simdjson_really_inline bool check_next_input(const __m512i input) {
|
||||
const __m512i v_80 = _mm512_set1_epi8(char(0x80));
|
||||
const __mmask64 ascii = _mm512_test_epi8_mask(input, v_80);
|
||||
if(ascii == 0) {
|
||||
this->error = _mm512_or_si512(this->error, this->prev_incomplete);
|
||||
return true;
|
||||
} else {
|
||||
this->check_utf8_bytes(input, this->prev_input_block);
|
||||
this->prev_incomplete = is_incomplete(input);
|
||||
this->prev_input_block = input;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// do not forget to call check_eof!
|
||||
simdjson_really_inline error_code errors() {
|
||||
return (_mm512_test_epi8_mask(this->error, this->error) != 0) ? error_code::UTF8_ERROR : error_code::SUCCESS;
|
||||
}
|
||||
}; // struct utf8_checker
|
||||
} // namespace utf8_validation
|
||||
|
||||
using utf8_validation::utf8_checker;
|
||||
|
||||
} // unnamed namespace
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
+65
-29
@@ -14,20 +14,41 @@ namespace internal {
|
||||
// Static array of known implementations. We're hoping these get baked into the executable
|
||||
// without requiring a static initializer.
|
||||
|
||||
#if SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
static const icelake::implementation* get_icelake_singleton() {
|
||||
static const icelake::implementation icelake_singleton{};
|
||||
return &icelake_singleton;
|
||||
}
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
const haswell::implementation haswell_singleton{};
|
||||
static const haswell::implementation* get_haswell_singleton() {
|
||||
static const haswell::implementation haswell_singleton{};
|
||||
return &haswell_singleton;
|
||||
}
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
const westmere::implementation westmere_singleton{};
|
||||
static const westmere::implementation* get_westmere_singleton() {
|
||||
static const westmere::implementation westmere_singleton{};
|
||||
return &westmere_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64
|
||||
const arm64::implementation arm64_singleton{};
|
||||
static const arm64::implementation* get_arm64_singleton() {
|
||||
static const arm64::implementation arm64_singleton{};
|
||||
return &arm64_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ARM64
|
||||
#if SIMDJSON_IMPLEMENTATION_PPC64
|
||||
const ppc64::implementation ppc64_singleton{};
|
||||
static const ppc64::implementation* get_ppc64_singleton() {
|
||||
static const ppc64::implementation ppc64_singleton{};
|
||||
return &ppc64_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_PPC64
|
||||
#if SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
const fallback::implementation fallback_singleton{};
|
||||
static const fallback::implementation* get_fallback_singleton() {
|
||||
static const fallback::implementation fallback_singleton{};
|
||||
return &fallback_singleton;
|
||||
}
|
||||
#endif // SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
|
||||
/**
|
||||
@@ -56,25 +77,29 @@ private:
|
||||
const implementation *set_best() const noexcept;
|
||||
};
|
||||
|
||||
const detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton;
|
||||
|
||||
const std::initializer_list<const implementation *> available_implementation_pointers {
|
||||
static const std::initializer_list<const implementation *>& get_available_implementation_pointers() {
|
||||
static const std::initializer_list<const implementation *> available_implementation_pointers {
|
||||
#if SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
get_icelake_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
&haswell_singleton,
|
||||
get_haswell_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_WESTMERE
|
||||
&westmere_singleton,
|
||||
get_westmere_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64
|
||||
&arm64_singleton,
|
||||
get_arm64_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_PPC64
|
||||
&ppc64_singleton,
|
||||
get_ppc64_singleton(),
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_FALLBACK
|
||||
&fallback_singleton,
|
||||
get_fallback_singleton(),
|
||||
#endif
|
||||
}; // available_implementation_pointers
|
||||
}; // available_implementation_pointers
|
||||
return available_implementation_pointers;
|
||||
}
|
||||
|
||||
// So we can return UNSUPPORTED_ARCHITECTURE from the parser when there is no support
|
||||
class unsupported_implementation final : public implementation {
|
||||
@@ -102,25 +127,28 @@ public:
|
||||
unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {}
|
||||
};
|
||||
|
||||
const unsupported_implementation unsupported_singleton{};
|
||||
const unsupported_implementation* get_unsupported_singleton() {
|
||||
static const unsupported_implementation unsupported_singleton{};
|
||||
return &unsupported_singleton;
|
||||
}
|
||||
|
||||
size_t available_implementation_list::size() const noexcept {
|
||||
return internal::available_implementation_pointers.size();
|
||||
return internal::get_available_implementation_pointers().size();
|
||||
}
|
||||
const implementation * const *available_implementation_list::begin() const noexcept {
|
||||
return internal::available_implementation_pointers.begin();
|
||||
return internal::get_available_implementation_pointers().begin();
|
||||
}
|
||||
const implementation * const *available_implementation_list::end() const noexcept {
|
||||
return internal::available_implementation_pointers.end();
|
||||
return internal::get_available_implementation_pointers().end();
|
||||
}
|
||||
const implementation *available_implementation_list::detect_best_supported() const noexcept {
|
||||
// They are prelisted in priority order, so we just go down the list
|
||||
uint32_t supported_instruction_sets = internal::detect_supported_architectures();
|
||||
for (const implementation *impl : internal::available_implementation_pointers) {
|
||||
for (const implementation *impl : internal::get_available_implementation_pointers()) {
|
||||
uint32_t required_instruction_sets = impl->required_instruction_sets();
|
||||
if ((supported_instruction_sets & required_instruction_sets) == required_instruction_sets) { return impl; }
|
||||
}
|
||||
return &unsupported_singleton; // this should never happen?
|
||||
return get_unsupported_singleton(); // this should never happen?
|
||||
}
|
||||
|
||||
const implementation *detect_best_supported_implementation_on_first_use::set_best() const noexcept {
|
||||
@@ -130,31 +158,39 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes
|
||||
SIMDJSON_POP_DISABLE_WARNINGS
|
||||
|
||||
if (force_implementation_name) {
|
||||
auto force_implementation = available_implementations[force_implementation_name];
|
||||
auto force_implementation = get_available_implementations()[force_implementation_name];
|
||||
if (force_implementation) {
|
||||
return active_implementation = force_implementation;
|
||||
return get_active_implementation() = force_implementation;
|
||||
} else {
|
||||
// Note: abort() and stderr usage within the library is forbidden.
|
||||
return active_implementation = &unsupported_singleton;
|
||||
return get_active_implementation() = get_unsupported_singleton();
|
||||
}
|
||||
}
|
||||
return active_implementation = available_implementations.detect_best_supported();
|
||||
return get_active_implementation() = get_available_implementations().detect_best_supported();
|
||||
}
|
||||
|
||||
} // namespace internal
|
||||
|
||||
SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{};
|
||||
SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton};
|
||||
SIMDJSON_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations() {
|
||||
static const internal::available_implementation_list available_implementations{};
|
||||
return available_implementations;
|
||||
}
|
||||
|
||||
SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation() {
|
||||
static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton;
|
||||
static internal::atomic_ptr<const implementation> active_implementation{&detect_best_supported_implementation_on_first_use_singleton};
|
||||
return active_implementation;
|
||||
}
|
||||
|
||||
simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, size_t &dst_len) noexcept {
|
||||
return active_implementation->minify(reinterpret_cast<const uint8_t *>(buf), len, reinterpret_cast<uint8_t *>(dst), dst_len);
|
||||
return get_active_implementation()->minify(reinterpret_cast<const uint8_t *>(buf), len, reinterpret_cast<uint8_t *>(dst), dst_len);
|
||||
}
|
||||
simdjson_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept {
|
||||
return active_implementation->validate_utf8(buf, len);
|
||||
return get_active_implementation()->validate_utf8(buf, len);
|
||||
}
|
||||
|
||||
const implementation * builtin_implementation() {
|
||||
static const implementation * builtin_impl = available_implementations[STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
|
||||
static const implementation * builtin_impl = get_available_implementations()[SIMDJSON_STRINGIFY(SIMDJSON_BUILTIN_IMPLEMENTATION)];
|
||||
assert(builtin_impl);
|
||||
return builtin_impl;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64
|
||||
#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
@@ -128,4 +128,4 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t thintable_epi8[256] = {
|
||||
} // namespace internal
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64
|
||||
#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64
|
||||
|
||||
@@ -19,6 +19,10 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
|
||||
#include "fallback/implementation.cpp"
|
||||
#include "fallback/dom_parser_implementation.cpp"
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_ICELAKE
|
||||
#include "icelake/implementation.cpp"
|
||||
#include "icelake/dom_parser_implementation.cpp"
|
||||
#endif
|
||||
#if SIMDJSON_IMPLEMENTATION_HASWELL
|
||||
#include "haswell/implementation.cpp"
|
||||
#include "haswell/dom_parser_implementation.cpp"
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
#include <cstring>
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
std::cout << "simdjson v" << STRINGIFY(SIMDJSON_VERSION) << " is running the " << simdjson::active_implementation->name() << " implementation." << std::endl;
|
||||
std::cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << " is running the " << simdjson::get_active_implementation()->name() << " implementation." << std::endl;
|
||||
const char *expected_implementation = nullptr;
|
||||
if (argc > 1) {
|
||||
expected_implementation = argv[1];
|
||||
@@ -15,7 +15,7 @@ int main(int argc, const char *argv[]) {
|
||||
}
|
||||
std::cout << "No expected implementation argument, but SIMDJSON_FORCE_IMPLEMENTATION is set to " << expected_implementation << ", so we'll check for that." << std::endl;
|
||||
}
|
||||
if (strcmp(expected_implementation, simdjson::active_implementation->name().c_str())) {
|
||||
if (strcmp(expected_implementation, simdjson::get_active_implementation()->name().c_str())) {
|
||||
std::cerr << "Wrong implementation! Expected " << expected_implementation << "." << std::endl;
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
#endif
|
||||
|
||||
const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793;
|
||||
#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, STRINGIFY(x))
|
||||
#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, SIMDJSON_STRINGIFY(x))
|
||||
|
||||
namespace number_tests {
|
||||
|
||||
@@ -1617,12 +1617,12 @@ namespace validate_tests {
|
||||
bool test_random() {
|
||||
std::cout << "Running " << __func__ << std::endl;
|
||||
std::vector<uint8_t> source(64,' ');
|
||||
const simdjson::implementation *impl_fallback = simdjson::available_implementations["fallback"];
|
||||
const simdjson::implementation *impl_fallback = simdjson::get_available_implementations()["fallback"];
|
||||
if(!impl_fallback) { return true; }
|
||||
for(size_t i = 0; i < 10000; i++) {
|
||||
std::vector<uint8_t>& s(source);
|
||||
s[i%64] ^= uint8_t(1235 * i);
|
||||
const bool active_ok = simdjson::active_implementation->validate_utf8((const char*)s.data(), s.size());
|
||||
const bool active_ok = simdjson::get_active_implementation()->validate_utf8((const char*)s.data(), s.size());
|
||||
const bool fallback_ok = impl_fallback->validate_utf8((const char*)s.data(), s.size());
|
||||
if(active_ok != fallback_ok) { return false; }
|
||||
s[i%64] ^= uint8_t(1235 * i);
|
||||
@@ -2137,7 +2137,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -2146,7 +2146,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -2156,12 +2156,12 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
// this is put here deliberately to check that the documentation is correct (README),
|
||||
// should this fail to compile, you should update the documentation:
|
||||
if (simdjson::active_implementation->name() == "unsupported") {
|
||||
if (simdjson::get_active_implementation()->name() == "unsupported") {
|
||||
printf("unsupported CPU\n");
|
||||
}
|
||||
// We want to know what we are testing.
|
||||
std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name();
|
||||
std::cout << " (" << simdjson::active_implementation->description() << ")" << std::endl;
|
||||
std::cout << "Running tests against this implementation: " << simdjson::get_active_implementation()->name();
|
||||
std::cout << " (" << simdjson::get_active_implementation()->description() << ")" << std::endl;
|
||||
std::cout << "------------------------------------------------------------" << std::endl;
|
||||
|
||||
std::cout << "Running basic tests." << std::endl;
|
||||
|
||||
@@ -952,7 +952,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -961,7 +961,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -972,12 +972,12 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
// this is put here deliberately to check that the documentation is correct (README),
|
||||
// should this fail to compile, you should update the documentation:
|
||||
if (simdjson::active_implementation->name() == "unsupported") {
|
||||
if (simdjson::get_active_implementation()->name() == "unsupported") {
|
||||
printf("unsupported CPU\n");
|
||||
}
|
||||
// We want to know what we are testing.
|
||||
std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name();
|
||||
std::cout << " (" << simdjson::active_implementation->description() << ")" << std::endl;
|
||||
std::cout << "Running tests against this implementation: " << simdjson::get_active_implementation()->name();
|
||||
std::cout << " (" << simdjson::get_active_implementation()->description() << ")" << std::endl;
|
||||
std::cout << "------------------------------------------------------------" << std::endl;
|
||||
|
||||
std::cout << "Running document_stream tests." << std::endl;
|
||||
|
||||
@@ -197,7 +197,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -206,7 +206,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -217,12 +217,12 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
// this is put here deliberately to check that the documentation is correct (README),
|
||||
// should this fail to compile, you should update the documentation:
|
||||
if (simdjson::active_implementation->name() == "unsupported") {
|
||||
if (simdjson::get_active_implementation()->name() == "unsupported") {
|
||||
printf("unsupported CPU\n");
|
||||
}
|
||||
// We want to know what we are testing.
|
||||
std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name();
|
||||
std::cout << " (" << simdjson::active_implementation->description() << ")" << std::endl;
|
||||
std::cout << "Running tests against this implementation: " << simdjson::get_active_implementation()->name();
|
||||
std::cout << " (" << simdjson::get_active_implementation()->description() << ")" << std::endl;
|
||||
std::cout << "------------------------------------------------------------" << std::endl;
|
||||
|
||||
std::cout << "Running document tests." << std::endl;
|
||||
|
||||
@@ -194,7 +194,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -203,7 +203,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -214,12 +214,12 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
// this is put here deliberately to check that the documentation is correct (README),
|
||||
// should this fail to compile, you should update the documentation:
|
||||
if (simdjson::active_implementation->name() == "unsupported") {
|
||||
if (simdjson::get_active_implementation()->name() == "unsupported") {
|
||||
printf("unsupported CPU\n");
|
||||
}
|
||||
// We want to know what we are testing.
|
||||
std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name();
|
||||
std::cout << " (" << simdjson::active_implementation->description() << ")" << std::endl;
|
||||
std::cout << "Running tests against this implementation: " << simdjson::get_active_implementation()->name();
|
||||
std::cout << " (" << simdjson::get_active_implementation()->description() << ")" << std::endl;
|
||||
std::cout << "------------------------------------------------------------" << std::endl;
|
||||
std::cout << "Running error tests." << std::endl;
|
||||
if (!(true
|
||||
|
||||
@@ -117,7 +117,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -126,7 +126,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -115,7 +115,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -124,7 +124,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
|
||||
@@ -158,7 +158,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -167,7 +167,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -178,12 +178,12 @@ int main(int argc, char *argv[]) {
|
||||
|
||||
// this is put here deliberately to check that the documentation is correct (README),
|
||||
// should this fail to compile, you should update the documentation:
|
||||
if (simdjson::active_implementation->name() == "unsupported") {
|
||||
if (simdjson::get_active_implementation()->name() == "unsupported") {
|
||||
printf("unsupported CPU\n");
|
||||
}
|
||||
// We want to know what we are testing.
|
||||
std::cout << "Running tests against this implementation: " << simdjson::active_implementation->name();
|
||||
std::cout << " (" << simdjson::active_implementation->description() << ")" << std::endl;
|
||||
std::cout << "Running tests against this implementation: " << simdjson::get_active_implementation()->name();
|
||||
std::cout << " (" << simdjson::get_active_implementation()->description() << ")" << std::endl;
|
||||
std::cout << "------------------------------------------------------------" << std::endl;
|
||||
if(optind >= argc) {
|
||||
std::cerr << "Usage: " << argv[0] << " <directorywithjsonfiles>"
|
||||
|
||||
@@ -158,7 +158,7 @@ int main(int argc, char *argv[]) {
|
||||
while ((c = getopt(argc, argv, "a:m:h")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
@@ -167,7 +167,7 @@ int main(int argc, char *argv[]) {
|
||||
fprintf(stderr, "The selected implementation does not match your current CPU: -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
case 'h': {
|
||||
|
||||
@@ -263,9 +263,9 @@ void basics_ndjson_parse_many() {
|
||||
}
|
||||
}
|
||||
void implementation_selection_1() {
|
||||
cout << "simdjson v" << STRINGIFY(SIMDJSON_VERSION) << endl;
|
||||
cout << "Detected the best implementation for your machine: " << simdjson::active_implementation->name();
|
||||
cout << "(" << simdjson::active_implementation->description() << ")" << endl;
|
||||
cout << "simdjson v" << SIMDJSON_STRINGIFY(SIMDJSON_VERSION) << endl;
|
||||
cout << "Detected the best implementation for your machine: " << simdjson::get_active_implementation()->name();
|
||||
cout << "(" << simdjson::get_active_implementation()->description() << ")" << endl;
|
||||
}
|
||||
|
||||
void unescaped_key() {
|
||||
@@ -288,32 +288,32 @@ void unescaped_key() {
|
||||
}
|
||||
|
||||
void implementation_selection_2() {
|
||||
for (auto implementation : simdjson::available_implementations) {
|
||||
for (auto implementation : simdjson::get_available_implementations()) {
|
||||
cout << implementation->name() << ": " << implementation->description() << endl;
|
||||
}
|
||||
}
|
||||
|
||||
void implementation_selection_2_safe() {
|
||||
for (auto implementation : simdjson::available_implementations) {
|
||||
for (auto implementation : simdjson::get_available_implementations()) {
|
||||
if(implementation->supported_by_runtime_system()) {
|
||||
cout << implementation->name() << ": " << implementation->description() << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
void implementation_selection_3() {
|
||||
cout << simdjson::available_implementations["fallback"]->description() << endl;
|
||||
cout << simdjson::get_available_implementations()["fallback"]->description() << endl;
|
||||
}
|
||||
|
||||
void implementation_selection_safe() {
|
||||
auto my_implementation = simdjson::available_implementations["haswell"];
|
||||
auto my_implementation = simdjson::get_available_implementations()["haswell"];
|
||||
if(! my_implementation) { exit(1); }
|
||||
if(! my_implementation->supported_by_runtime_system()) { exit(1); }
|
||||
simdjson::active_implementation = my_implementation;
|
||||
simdjson::get_active_implementation() = my_implementation;
|
||||
}
|
||||
|
||||
void implementation_selection_4() {
|
||||
// Use the fallback implementation, even though my machine is fast enough for anything
|
||||
simdjson::active_implementation = simdjson::available_implementations["fallback"];
|
||||
simdjson::get_active_implementation() = simdjson::get_available_implementations()["fallback"];
|
||||
}
|
||||
|
||||
void ondemand_performance_1() {
|
||||
|
||||
@@ -156,7 +156,7 @@ namespace document_stream_tests {
|
||||
"pd": 60,
|
||||
"sz": 3,
|
||||
"l": [
|
||||
"watts (est. ligth) ",
|
||||
"watts (est. light) ",
|
||||
"watts (1.9~gpm) ",
|
||||
"watts (1gpm) "
|
||||
],
|
||||
|
||||
@@ -133,7 +133,7 @@ namespace number_in_string_tests {
|
||||
ASSERT_SUCCESS(parser.iterate(int_doc).get(doc));
|
||||
ASSERT_SUCCESS(doc.get_int64_in_string().get(i));
|
||||
ASSERT_EQUAL(i,-243);
|
||||
// Unsinged integer
|
||||
// Unsigned integer
|
||||
ASSERT_SUCCESS(parser.iterate(uint_doc).get(doc));
|
||||
ASSERT_SUCCESS(doc.get_uint64_in_string().get(u));
|
||||
ASSERT_EQUAL(u,212213);
|
||||
|
||||
@@ -6,6 +6,53 @@ using namespace simdjson;
|
||||
namespace object_tests {
|
||||
using namespace std;
|
||||
using simdjson::ondemand::json_type;
|
||||
bool issue1745() {
|
||||
TEST_START();
|
||||
auto json = R"({
|
||||
"keys": [
|
||||
{
|
||||
"kid": "bbd2ac7c4c5eb8adc8eeffbc8f5a2dd6cf7545e4",
|
||||
"e": "AQAB",
|
||||
"alg": "RS256",
|
||||
"use": "sig",
|
||||
"n": "y930dtGTeMG52IPsKmMuEpPHLaxuYQlduZd6BqFVjc2-UFZR8fNqtnYzAjbXWJD_Tqxgdlj_MW4vogvX4sHwVpZONvdyeGoIyDQtis6iuGQhQamV85F_JbrEUnEw3QCO87Liz5UXG6BK2HRyPhDfMex1_tO0ROmySLFdCTS17D0wah71Ibpi0gI8LUi6kzVRjYDIC1oE-iK3Y9s88Bi4ZGYJxXAbnNwbwVkGOKCXja9k0jjBGRxZD-4KDuf493lFOOEGSLDA2Qp9rDqrURP12XYgvf_zJx_kSDipnr0gL6Vz2n3H4-XN4tA45zuzRkHoE7-XexPq-tv7kQ8pSjY2uQ",
|
||||
"kty": "RSA"
|
||||
},
|
||||
{
|
||||
"kty": "RSA",
|
||||
"n": "zMHxWuxztMKXdBhv3rImlUvW_yp6nO03cVXPyA0Vyq0-M7LfOJJIF-OdNoRGdsFPHVKCFoo6qGhR8rBCmMxA4fM-Ubk5qKuUqCN9eP3yZJq8Cw9tUrt_qh7uW-qfMr0upcyeSHhC_zW1lTGs5sowDorKN_jQ1Sfh9hfBxfc8T7dQAAgEqqMcE3u-2J701jyhJz0pvurCfziiB3buY6SGREhBQwNwpnQjt_lE2U4km8FS0woPzt0ccE3zsGL2qM-LWZbOm9aXquSnqNJLt3tGVvShnev-GiJ1XfQ3EWm0f4w0TX9fTOkxstl0vo_vW_FjGQ0D1pXSjqb7n-hAdXwc9w",
|
||||
"alg": "RS256",
|
||||
"kid": "85828c59284a69b54b27483e487c3bd46cd2a2b3",
|
||||
"e": "AQAB",
|
||||
"use": "sig"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
})"_padded;
|
||||
ondemand::parser parser;
|
||||
ondemand::document doc;
|
||||
ASSERT_SUCCESS(parser.iterate(json).get(doc));
|
||||
for (auto items_instance : doc["keys"]) {
|
||||
ondemand::object items;
|
||||
ASSERT_SUCCESS(items_instance.get_object().get(items));
|
||||
std::string_view kty_value;
|
||||
ASSERT_SUCCESS(items["kty"].get_string().get(kty_value));
|
||||
std::string_view use_value;
|
||||
ASSERT_SUCCESS(items["use"].get_string().get(use_value));
|
||||
if (kty_value != "RSA" && use_value != "sig") { continue; }
|
||||
std::string_view str;
|
||||
ASSERT_SUCCESS(items["alg"].get_string().get(str));
|
||||
ASSERT_EQUAL(str, "RS256");
|
||||
ASSERT_EQUAL(use_value, "sig");
|
||||
ASSERT_EQUAL(kty_value, "RSA");
|
||||
ASSERT_SUCCESS(items["n"].get_string().get(str));
|
||||
ASSERT_EQUAL(str.size(), 342);
|
||||
ASSERT_SUCCESS(items["e"].get_string().get(str));
|
||||
ASSERT_EQUAL(str, "AQAB");
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool issue1723() {
|
||||
TEST_START();
|
||||
@@ -178,6 +225,47 @@ namespace object_tests {
|
||||
}
|
||||
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
bool issue1745_with_exceptions() {
|
||||
TEST_START();
|
||||
auto json = R"({
|
||||
"keys": [
|
||||
{
|
||||
"kid": "bbd2ac7c4c5eb8adc8eeffbc8f5a2dd6cf7545e4",
|
||||
"e": "AQAB",
|
||||
"alg": "RS256",
|
||||
"use": "sig",
|
||||
"n": "y930dtGTeMG52IPsKmMuEpPHLaxuYQlduZd6BqFVjc2-UFZR8fNqtnYzAjbXWJD_Tqxgdlj_MW4vogvX4sHwVpZONvdyeGoIyDQtis6iuGQhQamV85F_JbrEUnEw3QCO87Liz5UXG6BK2HRyPhDfMex1_tO0ROmySLFdCTS17D0wah71Ibpi0gI8LUi6kzVRjYDIC1oE-iK3Y9s88Bi4ZGYJxXAbnNwbwVkGOKCXja9k0jjBGRxZD-4KDuf493lFOOEGSLDA2Qp9rDqrURP12XYgvf_zJx_kSDipnr0gL6Vz2n3H4-XN4tA45zuzRkHoE7-XexPq-tv7kQ8pSjY2uQ",
|
||||
"kty": "RSA"
|
||||
},
|
||||
{
|
||||
"kty": "RSA",
|
||||
"n": "zMHxWuxztMKXdBhv3rImlUvW_yp6nO03cVXPyA0Vyq0-M7LfOJJIF-OdNoRGdsFPHVKCFoo6qGhR8rBCmMxA4fM-Ubk5qKuUqCN9eP3yZJq8Cw9tUrt_qh7uW-qfMr0upcyeSHhC_zW1lTGs5sowDorKN_jQ1Sfh9hfBxfc8T7dQAAgEqqMcE3u-2J701jyhJz0pvurCfziiB3buY6SGREhBQwNwpnQjt_lE2U4km8FS0woPzt0ccE3zsGL2qM-LWZbOm9aXquSnqNJLt3tGVvShnev-GiJ1XfQ3EWm0f4w0TX9fTOkxstl0vo_vW_FjGQ0D1pXSjqb7n-hAdXwc9w",
|
||||
"alg": "RS256",
|
||||
"kid": "85828c59284a69b54b27483e487c3bd46cd2a2b3",
|
||||
"e": "AQAB",
|
||||
"use": "sig"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
})"_padded;
|
||||
ondemand::parser parser;
|
||||
ondemand::document doc = parser.iterate(json);
|
||||
for (auto items : doc["keys"]) {
|
||||
std::string_view kty_value = items["kty"].get_string();
|
||||
std::string_view use_value = items["use"].get_string();
|
||||
if (kty_value != "RSA" && use_value != "sig") { continue; }
|
||||
std::string_view str = items["alg"].get_string();
|
||||
if(str != "RS256") { return false; }
|
||||
if(use_value != "sig") { return false; }
|
||||
if(kty_value != "RSA") { return false; }
|
||||
str = items["n"].get_string();
|
||||
if(str.size() != 342) { return false; }
|
||||
str = items["e"].get_string();
|
||||
if(str != "AQAB") { return false; }
|
||||
}
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool issue1723_except() {
|
||||
TEST_START();
|
||||
@@ -988,6 +1076,7 @@ namespace object_tests {
|
||||
|
||||
bool run() {
|
||||
return
|
||||
issue1745() &&
|
||||
issue1742() &&
|
||||
issue1742_value() &&
|
||||
issue1723() &&
|
||||
@@ -997,6 +1086,7 @@ namespace object_tests {
|
||||
missing_keys() &&
|
||||
missing_keys_for_empty_top_level_object() &&
|
||||
#if SIMDJSON_EXCEPTIONS
|
||||
issue1745_with_exceptions() &&
|
||||
issue1723_except() &&
|
||||
fixed_broken_issue_1521() &&
|
||||
issue_1521() &&
|
||||
|
||||
@@ -13,6 +13,26 @@
|
||||
#include <vector>
|
||||
|
||||
#include "simdjson.h"
|
||||
|
||||
|
||||
/**
|
||||
* What follows is merely a compilation test to verify
|
||||
* https://github.com/simdjson/simdjson/issues/1768
|
||||
* It deliberately resides before any namespace declaration
|
||||
* within this file.
|
||||
*/
|
||||
namespace issue1768 {
|
||||
template <typename T> void print(const T &a) { std::cout << a; }
|
||||
} // namespace NA
|
||||
|
||||
void issue1768_test() {
|
||||
simdjson::ondemand::object obj;
|
||||
issue1768::print(obj);
|
||||
}
|
||||
/**
|
||||
* End of 1768 compilation check.
|
||||
*/
|
||||
|
||||
using namespace simdjson;
|
||||
|
||||
#include "test_ondemand.h"
|
||||
@@ -438,6 +458,9 @@ bool run() {
|
||||
|
||||
} // namespace tostring_tests
|
||||
|
||||
|
||||
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
return test_main(argc, argv, tostring_tests::run);
|
||||
}
|
||||
|
||||
@@ -41,7 +41,7 @@ bool test_ondemand_doc(const simdjson::padded_string &json, const F& f) {
|
||||
}
|
||||
|
||||
const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793;
|
||||
#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, STRINGIFY(x))
|
||||
#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, SIMDJSON_STRINGIFY(x))
|
||||
|
||||
template<typename F>
|
||||
int test_main(int argc, char *argv[], const F& test_function) {
|
||||
@@ -50,12 +50,12 @@ int test_main(int argc, char *argv[], const F& test_function) {
|
||||
while ((c = getopt(argc, argv, "a:")) != -1) {
|
||||
switch (c) {
|
||||
case 'a': {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[optarg];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
|
||||
if (!impl) {
|
||||
std::fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
@@ -66,7 +66,7 @@ int test_main(int argc, char *argv[], const F& test_function) {
|
||||
|
||||
// this is put here deliberately to check that the documentation is correct (README),
|
||||
// should this fail to compile, you should update the documentation:
|
||||
if (simdjson::active_implementation->name() == "unsupported") {
|
||||
if (simdjson::get_active_implementation()->name() == "unsupported") {
|
||||
std::printf("unsupported CPU\n");
|
||||
std::abort();
|
||||
}
|
||||
|
||||
@@ -255,7 +255,7 @@ void puzzler() {
|
||||
}
|
||||
std::cout << "\"" << std::endl;
|
||||
bool is_ok{true};
|
||||
for(const auto& e: simdjson::available_implementations) {
|
||||
for(const auto& e: simdjson::get_available_implementations()) {
|
||||
if(!e->supported_by_runtime_system()) { continue; }
|
||||
const bool current = e->validate_utf8(bad64, length);
|
||||
std::cout << e->name() << " returns " << current << std::endl;
|
||||
|
||||
+2
-2
@@ -20,9 +20,9 @@ int main(int argc, const char *argv[]) {
|
||||
std::string progName = "json2json";
|
||||
|
||||
std::string progUsage = "json2json version ";
|
||||
progUsage += STRINGIFY(SIMDJSON_VERSION);
|
||||
progUsage += SIMDJSON_STRINGIFY(SIMDJSON_VERSION);
|
||||
progUsage += " (";
|
||||
progUsage += simdjson::active_implementation->name();
|
||||
progUsage += simdjson::get_active_implementation()->name();
|
||||
progUsage += ")\n";
|
||||
progUsage += "Reads json in, out the result of the parsing.\n";
|
||||
progUsage += argv[0];
|
||||
|
||||
+4
-4
@@ -31,7 +31,7 @@ int main(int argc, const char *argv[]) {
|
||||
std::stringstream ss;
|
||||
ss << "Parser implementation (by default, detects the most advanced implementation supported on the host machine)." << std::endl;
|
||||
ss << "Available parser implementations:" << std::endl;
|
||||
for (auto impl : simdjson::available_implementations) {
|
||||
for (auto impl : simdjson::get_available_implementations()) {
|
||||
if(impl->supported_by_runtime_system()) {
|
||||
ss << "-a " << std::left << std::setw(9) << impl->name() << " - Use the " << impl->description() << " parser implementation." << std::endl;
|
||||
}
|
||||
@@ -55,7 +55,7 @@ int main(int argc, const char *argv[]) {
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
if(result.count("arch")) {
|
||||
const simdjson::implementation *impl = simdjson::available_implementations[result["arch"].as<std::string>().c_str()];
|
||||
const simdjson::implementation *impl = simdjson::get_available_implementations()[result["arch"].as<std::string>().c_str()];
|
||||
if(!impl) {
|
||||
usage("Unsupported implementation.");
|
||||
return EXIT_FAILURE;
|
||||
@@ -64,7 +64,7 @@ int main(int argc, const char *argv[]) {
|
||||
usage("The selected implementation does not match your current CPU.");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
simdjson::active_implementation = impl;
|
||||
simdjson::get_active_implementation() = impl;
|
||||
}
|
||||
|
||||
std::string filename = result["file"].as<std::string>();
|
||||
@@ -77,7 +77,7 @@ int main(int argc, const char *argv[]) {
|
||||
}
|
||||
simdjson::padded_string copy(p.length()); // does not need to be padded after all!
|
||||
size_t copy_len;
|
||||
error = simdjson::active_implementation->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
|
||||
error = simdjson::get_active_implementation()->minify((const uint8_t*)p.data(), p.length(), (uint8_t*)copy.data(), copy_len);
|
||||
if (error) { std::cerr << error << std::endl; return EXIT_FAILURE; }
|
||||
printf("%s", copy.data());
|
||||
return EXIT_SUCCESS;
|
||||
|
||||
Reference in New Issue
Block a user