mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
Compare commits
88 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 450cb7b0f8 | |||
| 2907912b1d | |||
| b6aff44c42 | |||
| ce11f41840 | |||
| 7c8bc650a7 | |||
| 657489e387 | |||
| 60b6a16edc | |||
| 2ceca3e510 | |||
| d8b6a8c9d3 | |||
| bbc9bd059a | |||
| 68df233b4b | |||
| dc66d6133a | |||
| 13055a4601 | |||
| a7ff410532 | |||
| d72cba29be | |||
| 757db1b2f0 | |||
| 7175731e09 | |||
| f6fc786b05 | |||
| 8ce1e52e4f | |||
| b12014f9e4 | |||
| 1a433fc5b4 | |||
| 21845f1f6a | |||
| dc86daa4ec | |||
| 910686ffa8 | |||
| b1a3943a8b | |||
| 70a0d62f51 | |||
| bb0ed22ab8 | |||
| d7770e9a6b | |||
| 0e8b68683d | |||
| 1fe50cb777 | |||
| f2d419dda6 | |||
| 57ca29e151 | |||
| 24068a089e | |||
| e4bba2df9d | |||
| ca9391c5a9 | |||
| 91b8fa4c4b | |||
| 081c3ca7c8 | |||
| bbeec66a93 | |||
| 7dcd43e82a | |||
| c3f7eff019 | |||
| d0e2a7fb00 | |||
| 9cace04da3 | |||
| 2e688a2825 | |||
| 4501aebee8 | |||
| df24da314f | |||
| 8b2bff981f | |||
| a34996939c | |||
| e8e731a311 | |||
| f364e354e2 | |||
| 3c5c2d2404 | |||
| 3300d8463b | |||
| 267e70d550 | |||
| 4b118bd9e1 | |||
| e5d0e92116 | |||
| b3b108b120 | |||
| 40f522ab2d | |||
| 5a6894cd3e | |||
| 90409897eb | |||
| 78d1cd96a9 | |||
| 6a1f8fc5c3 | |||
| 42a31ad096 | |||
| ac647e6c11 | |||
| 45fbd8c4c5 | |||
| 6ba9ad3ed9 | |||
| bab51e5f08 | |||
| f45c59f27c | |||
| ba0406f021 | |||
| b02aedb5bb | |||
| da4de2e0b8 | |||
| eda0f190ea | |||
| 633784a4a2 | |||
| 10a7b9bd25 | |||
| 0d8f6f5827 | |||
| 0c14707b78 | |||
| c0fb5f7474 | |||
| 930b0beb6d | |||
| dadfb519cd | |||
| 595c628965 | |||
| 76f7e4ae04 | |||
| d7b08c84c7 | |||
| 4d0f1021ff | |||
| 42c854124f | |||
| 081f954465 | |||
| 1fbd6047c0 | |||
| 09601eb480 | |||
| 02762fa352 | |||
| a334303119 | |||
| 7f545a8996 |
+5
-5
@@ -15,22 +15,22 @@ environment:
|
|||||||
- job_name: VS2019
|
- job_name: VS2019
|
||||||
CMAKE_ARGS: -A %Platform%
|
CMAKE_ARGS: -A %Platform%
|
||||||
- job_name: VS2019ARM
|
- job_name: VS2019ARM
|
||||||
CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
|
CMAKE_ARGS: -A ARM64 -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM?
|
||||||
- job_name: VS2017 (Static, No Threads)
|
- job_name: VS2017 (Static, No Threads)
|
||||||
image: Visual Studio 2017
|
image: Visual Studio 2017
|
||||||
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
|
CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
|
||||||
CTEST_ARGS: -LE explicitonly
|
CTEST_ARGS: -LE explicitonly
|
||||||
- job_name: VS2019 (Win32)
|
- job_name: VS2019 (Win32)
|
||||||
platform: Win32
|
platform: Win32
|
||||||
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
|
CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway.
|
||||||
CTEST_ARGS: -LE explicitonly
|
CTEST_ARGS: -LE explicitonly
|
||||||
- job_name: VS2019 (Win32, No Exceptions)
|
- job_name: VS2019 (Win32, No Exceptions)
|
||||||
platform: Win32
|
platform: Win32
|
||||||
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
|
CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF
|
||||||
CTEST_ARGS: -LE explicitonly
|
CTEST_ARGS: -LE explicitonly
|
||||||
- job_name: VS2015
|
- job_name: VS2015
|
||||||
image: Visual Studio 2015
|
image: Visual Studio 2015
|
||||||
CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
|
CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF
|
||||||
CTEST_ARGS: -LE explicitonly
|
CTEST_ARGS: -LE explicitonly
|
||||||
|
|
||||||
build_script:
|
build_script:
|
||||||
|
|||||||
@@ -88,13 +88,13 @@ commands:
|
|||||||
steps:
|
steps:
|
||||||
- cmake_prep
|
- cmake_prep
|
||||||
- dependency_restore
|
- dependency_restore
|
||||||
- run: cmake $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build .
|
- run: cmake -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build .
|
||||||
- dependency_cache # dependencies are produced in the configure step
|
- dependency_cache # dependencies are produced in the configure step
|
||||||
|
|
||||||
cmake_build:
|
cmake_build:
|
||||||
steps:
|
steps:
|
||||||
- cmake_build_cache
|
- cmake_build_cache
|
||||||
- run: cmake --build build
|
- run: cmake --build build
|
||||||
|
|
||||||
cmake_test:
|
cmake_test:
|
||||||
steps:
|
steps:
|
||||||
@@ -191,12 +191,12 @@ jobs:
|
|||||||
sanitize-gcc10:
|
sanitize-gcc10:
|
||||||
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
|
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
|
||||||
executor: gcc10
|
executor: gcc10
|
||||||
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
environment: { CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
||||||
steps: [ cmake_test ]
|
steps: [ cmake_test ]
|
||||||
sanitize-clang10:
|
sanitize-clang10:
|
||||||
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
|
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
|
||||||
executor: clang10
|
executor: clang10
|
||||||
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
||||||
steps: [ cmake_test ]
|
steps: [ cmake_test ]
|
||||||
threadsanitize-gcc10:
|
threadsanitize-gcc10:
|
||||||
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
|
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
|
||||||
@@ -206,7 +206,7 @@ jobs:
|
|||||||
threadsanitize-clang10:
|
threadsanitize-clang10:
|
||||||
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
|
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
|
||||||
executor: clang10
|
executor: clang10
|
||||||
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
||||||
steps: [ cmake_test ]
|
steps: [ cmake_test ]
|
||||||
nocheckeof-clang10:
|
nocheckeof-clang10:
|
||||||
description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
|
description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds
|
||||||
@@ -270,12 +270,12 @@ jobs:
|
|||||||
sanitize-haswell-gcc10:
|
sanitize-haswell-gcc10:
|
||||||
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
|
description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build
|
||||||
executor: gcc10
|
executor: gcc10
|
||||||
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
||||||
steps: [ cmake_test ]
|
steps: [ cmake_test ]
|
||||||
sanitize-haswell-clang10:
|
sanitize-haswell-clang10:
|
||||||
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
|
description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build
|
||||||
executor: clang10
|
executor: clang10
|
||||||
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly }
|
||||||
steps: [ cmake_test ]
|
steps: [ cmake_test ]
|
||||||
|
|
||||||
workflows:
|
workflows:
|
||||||
@@ -292,7 +292,7 @@ workflows:
|
|||||||
- libcpp-clang10
|
- libcpp-clang10
|
||||||
|
|
||||||
# full single-implementation tests
|
# full single-implementation tests
|
||||||
- sanitize-gcc10
|
# - sanitize-gcc10 # gcc10 + sanitize uses a lot of memory?
|
||||||
- sanitize-clang10
|
- sanitize-clang10
|
||||||
- threadsanitize-gcc10
|
- threadsanitize-gcc10
|
||||||
- threadsanitize-clang10
|
- threadsanitize-clang10
|
||||||
@@ -311,7 +311,7 @@ workflows:
|
|||||||
|
|
||||||
|
|
||||||
# sanitized single-implementation tests
|
# sanitized single-implementation tests
|
||||||
- sanitize-haswell-gcc10
|
# - sanitize-haswell-gcc10 # gcc10 + sanitize uses a lot of memory?
|
||||||
- sanitize-haswell-clang10
|
- sanitize-haswell-clang10
|
||||||
|
|
||||||
# testing "just the library"
|
# testing "just the library"
|
||||||
|
|||||||
+2
-2
@@ -19,8 +19,8 @@ task:
|
|||||||
build_script:
|
build_script:
|
||||||
- mkdir build
|
- mkdir build
|
||||||
- cd build
|
- cd build
|
||||||
- cmake -DSIMDJSON_BASH=OFF -DSIMDJSON_GIT=OFF ..
|
- cmake -DSIMDJSON_BASH=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_GIT=OFF ..
|
||||||
- make
|
- cmake --build .
|
||||||
test_script:
|
test_script:
|
||||||
- cd build
|
- cd build
|
||||||
- ctest --output-on-failure -LE explicitonly
|
- ctest --output-on-failure -LE explicitonly
|
||||||
|
|||||||
+19
-19
@@ -8,7 +8,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
|
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DEVELOPER_MODE=ON
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
|
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
|
||||||
@@ -29,7 +29,7 @@ steps:
|
|||||||
CC: clang-6.0
|
CC: clang-6.0
|
||||||
CXX: clang++-6.0
|
CXX: clang++-6.0
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
|
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DEVELOPER_MODE=ON
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
|
- scripts/addcmakeppa.sh "$(env -i sh -c '. /etc/os-release; echo $VERSION_CODENAME')"
|
||||||
@@ -50,7 +50,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||||
@@ -77,7 +77,7 @@ steps:
|
|||||||
CC: clang-6.0
|
CC: clang-6.0
|
||||||
CXX: clang++-6.0
|
CXX: clang++-6.0
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- mkdir build
|
- mkdir build
|
||||||
@@ -100,7 +100,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||||
@@ -122,7 +122,7 @@ steps:
|
|||||||
environment:
|
environment:
|
||||||
CC: clang-9
|
CC: clang-9
|
||||||
CXX: clang++-9
|
CXX: clang++-9
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
@@ -142,7 +142,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||||
@@ -168,7 +168,7 @@ steps:
|
|||||||
environment:
|
environment:
|
||||||
CC: clang-9
|
CC: clang-9
|
||||||
CXX: clang++-9
|
CXX: clang++-9
|
||||||
CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=haswell;westmere;fallback
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
@@ -192,7 +192,7 @@ steps:
|
|||||||
environment:
|
environment:
|
||||||
CC: clang-11
|
CC: clang-11
|
||||||
CXX: clang++-11
|
CXX: clang++-11
|
||||||
CMAKE_FLAGS: -GNinja
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -GNinja
|
||||||
BUILD_FLAGS:
|
BUILD_FLAGS:
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
CXXFLAGS: -std=c++20 -stdlib=libc++
|
CXXFLAGS: -std=c++20 -stdlib=libc++
|
||||||
@@ -213,7 +213,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||||
@@ -237,7 +237,7 @@ steps:
|
|||||||
environment:
|
environment:
|
||||||
CC: clang-6.0
|
CC: clang-6.0
|
||||||
CXX: clang++-6.0
|
CXX: clang++-6.0
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
@@ -260,7 +260,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||||
@@ -281,7 +281,7 @@ steps:
|
|||||||
environment:
|
environment:
|
||||||
CC: clang-6.0
|
CC: clang-6.0
|
||||||
CXX: clang++-6.0
|
CXX: clang++-6.0
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
@@ -302,7 +302,7 @@ steps:
|
|||||||
image: gcc:8
|
image: gcc:8
|
||||||
environment:
|
environment:
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_IMPLEMENTATION=arm64;fallback
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
@@ -329,7 +329,7 @@ steps:
|
|||||||
environment:
|
environment:
|
||||||
CC: clang-6.0
|
CC: clang-6.0
|
||||||
CXX: clang++-6.0
|
CXX: clang++-6.0
|
||||||
CMAKE_FLAGS: -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON -DSIMDJSON_IMPLEMENTATION=arm64;fallback
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
@@ -356,7 +356,7 @@ steps:
|
|||||||
CC: clang-9
|
CC: clang-9
|
||||||
CXX: clang++-9
|
CXX: clang++-9
|
||||||
BUILD_FLAGS: -- -j 4
|
BUILD_FLAGS: -- -j 4
|
||||||
CMAKE_FLAGS: -GNinja -DBUILD_SHARED_LIBS=OFF
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -GNinja -DBUILD_SHARED_LIBS=OFF
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
CXXFLAGS: -stdlib=libc++
|
CXXFLAGS: -stdlib=libc++
|
||||||
commands:
|
commands:
|
||||||
@@ -377,7 +377,7 @@ steps:
|
|||||||
CC: clang-9
|
CC: clang-9
|
||||||
CXX: clang++-9
|
CXX: clang++-9
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
CXXFLAGS: -stdlib=libc++
|
CXXFLAGS: -stdlib=libc++
|
||||||
commands:
|
commands:
|
||||||
@@ -398,7 +398,7 @@ steps:
|
|||||||
CC: clang-7
|
CC: clang-7
|
||||||
CXX: clang++-7
|
CXX: clang++-7
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DBUILD_SHARED_LIBS=OFF
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
CXXFLAGS: -stdlib=libc++
|
CXXFLAGS: -stdlib=libc++
|
||||||
commands:
|
commands:
|
||||||
@@ -418,7 +418,7 @@ steps:
|
|||||||
CC: gcc
|
CC: gcc
|
||||||
CXX: g++
|
CXX: g++
|
||||||
BUILD_FLAGS: -- -j
|
BUILD_FLAGS: -- -j
|
||||||
CMAKE_FLAGS: -DSIMDJSON_EXCEPTIONS=OFF
|
CMAKE_FLAGS: -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_EXCEPTIONS=OFF
|
||||||
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
CTEST_FLAGS: -j4 --output-on-failure -LE explicitonly
|
||||||
commands:
|
commands:
|
||||||
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
- echo "deb http://deb.debian.org/debian buster-backports main" >> /etc/apt/sources.list
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: Alpine Linux
|
name: Alpine Linux
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -1,9 +1,6 @@
|
|||||||
name: Detect trailing whitespace
|
name: Detect trailing whitespace
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
whitespace:
|
whitespace:
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
name: MinGW32-CI
|
name: MinGW32-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
|
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
name: MinGW64-CI
|
name: MinGW64-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
|
# Important: scoop will either install 32-bit GCC or 64-bit GCC, not both.
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
name: MSYS2-CLANG-CI
|
name: MSYS2-CLANG-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
windows-mingw:
|
windows-mingw:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: MSYS2-CI
|
name: MSYS2-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
windows-mingw:
|
windows-mingw:
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer
|
name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: Ubuntu 18.04 CI (GCC 7)
|
name: Ubuntu 18.04 CI (GCC 7)
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: Ubuntu 20.04 CI (GCC 9) without exceptions
|
name: Ubuntu 20.04 CI (GCC 9) without exceptions
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: Ubuntu 20.04 CI (GCC 9) without threads
|
name: Ubuntu 20.04 CI (GCC 9) Without Threads
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
name: Ubuntu 20.04 CI (GCC 9) With Memory Sanitizer
|
||||||
|
|
||||||
|
on: [push, pull_request]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
ubuntu-build:
|
||||||
|
if: >-
|
||||||
|
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
|
||||||
|
! contains(toJSON(github.event.commits.*.message), '[skip github]')
|
||||||
|
runs-on: ubuntu-20.04
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v2
|
||||||
|
- uses: actions/cache@v2
|
||||||
|
with:
|
||||||
|
path: dependencies/.cache
|
||||||
|
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
|
||||||
|
- name: Use cmake
|
||||||
|
run: |
|
||||||
|
mkdir builddebug &&
|
||||||
|
cd builddebug &&
|
||||||
|
cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
|
||||||
|
cmake --build . &&
|
||||||
|
ctest -j --output-on-failure -LE explicitonly
|
||||||
@@ -1,12 +1,6 @@
|
|||||||
name: Ubuntu 20.04 CI (GCC 9) with Thread Sanitizer
|
name: Ubuntu 20.04 CI (GCC 9) with Thread Sanitizer
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: Ubuntu 20.04 CI (GCC 9)
|
name: Ubuntu 20.04 CI (GCC 9)
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu-build:
|
ubuntu-build:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: VS15-CI
|
name: VS15-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: VS16-CI
|
name: VS16-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: VS16-CLANG-CI
|
name: VS16-CLANG-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: VS16-Ninja-CI
|
name: VS16-Ninja-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
@@ -1,12 +1,6 @@
|
|||||||
name: VS16-NoExcept-CI
|
name: VS16-NoExcept-CI
|
||||||
|
|
||||||
on:
|
on: [push, pull_request]
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ci:
|
ci:
|
||||||
|
|||||||
+2
-2
@@ -172,7 +172,7 @@ install:
|
|||||||
- export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}"
|
- export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}"
|
||||||
- export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback";
|
- export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback";
|
||||||
- if [[ "${SANITIZE}" == "on" ]]; then
|
- if [[ "${SANITIZE}" == "on" ]]; then
|
||||||
export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_SANITIZE=ON";
|
export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON";
|
||||||
export ASAN_OPTIONS="detect_leaks=0";
|
export ASAN_OPTIONS="detect_leaks=0";
|
||||||
fi
|
fi
|
||||||
- if [[ "${STATIC}" == "on" ]]; then
|
- if [[ "${STATIC}" == "on" ]]; then
|
||||||
@@ -183,7 +183,7 @@ install:
|
|||||||
script:
|
script:
|
||||||
- mkdir build
|
- mkdir build
|
||||||
- cd build
|
- cd build
|
||||||
- cmake $CMAKE_FLAGS ..
|
- cmake -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS ..
|
||||||
- cmake --build . -- -j2
|
- cmake --build . -- -j2
|
||||||
- SIMDJSON_FORCE_IMPLEMENTATION=ppc64 ctest $CTEST_FLAGS -L per_implementation
|
- SIMDJSON_FORCE_IMPLEMENTATION=ppc64 ctest $CTEST_FLAGS -L per_implementation
|
||||||
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
|
- SIMDJSON_FORCE_IMPLEMENTATION=fallback ctest $CTEST_FLAGS -L per_implementation
|
||||||
|
|||||||
Vendored
+2
-2
@@ -17,7 +17,6 @@
|
|||||||
"__errc": "cpp",
|
"__errc": "cpp",
|
||||||
"__functional_base": "cpp",
|
"__functional_base": "cpp",
|
||||||
"__hash_table": "cpp",
|
"__hash_table": "cpp",
|
||||||
"__locale": "cpp",
|
|
||||||
"__mutex_base": "cpp",
|
"__mutex_base": "cpp",
|
||||||
"__node_handle": "cpp",
|
"__node_handle": "cpp",
|
||||||
"__nullptr": "cpp",
|
"__nullptr": "cpp",
|
||||||
@@ -85,6 +84,7 @@
|
|||||||
"utility": "cpp",
|
"utility": "cpp",
|
||||||
"valarray": "cpp",
|
"valarray": "cpp",
|
||||||
"vector": "cpp",
|
"vector": "cpp",
|
||||||
"*.ipp": "cpp"
|
"*.ipp": "cpp",
|
||||||
|
"filesystem": "cpp"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,6 +3,19 @@
|
|||||||
#
|
#
|
||||||
add_library(simdjson-internal-flags INTERFACE)
|
add_library(simdjson-internal-flags INTERFACE)
|
||||||
|
|
||||||
|
option(SIMDJSON_NO_FORCE_INLINING "Do not attempt to force function inlining" OFF)
|
||||||
|
if(SIMDJSON_NO_FORCE_INLINING)
|
||||||
|
add_compile_definitions(SIMDJSON_NO_FORCE_INLINING=1)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer (to avoid buffer overflows). We expect this setting to be ON except for performance testing." ON)
|
||||||
|
if(SIMDJSON_CHECK_EOF)
|
||||||
|
add_compile_definitions(SIMDJSON_CHECK_EOF=1)
|
||||||
|
else()
|
||||||
|
add_compile_definitions(SIMDJSON_CHECK_EOF=0)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
|
||||||
option(SIMDJSON_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF)
|
option(SIMDJSON_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF)
|
||||||
if(SIMDJSON_SANITIZE_UNDEFINED)
|
if(SIMDJSON_SANITIZE_UNDEFINED)
|
||||||
add_compile_options(-fsanitize=undefined -fno-sanitize-recover=all)
|
add_compile_options(-fsanitize=undefined -fno-sanitize-recover=all)
|
||||||
|
|||||||
+46
-49
@@ -132,25 +132,22 @@ auto json = padded_string::load("twitter.json");
|
|||||||
ondemand::document doc = parser.iterate(json); // position a pointer at the beginning of the JSON data
|
ondemand::document doc = parser.iterate(json); // position a pointer at the beginning of the JSON data
|
||||||
```
|
```
|
||||||
|
|
||||||
Or by creating a padded string (for efficiency reasons, simdjson requires a string with
|
Or by creating a string and calling `iterate()`:
|
||||||
SIMDJSON_PADDING bytes at the end) and calling `iterate()`:
|
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto json = "[1,2,3]"_padded; // The _padded suffix creates a simdjson::padded_string instance
|
std::string json = "[1,2,3]";
|
||||||
ondemand::document doc = parser.iterate(json); // parse a string
|
ondemand::document doc = parser.iterate(json); // parse a string
|
||||||
```
|
```
|
||||||
|
|
||||||
If you have a buffer of your own with enough padding already (SIMDJSON_PADDING extra bytes allocated), you can use `padded_string_view` to pass it in:
|
If you have a buffer of your own pass it in:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
char json[3+SIMDJSON_PADDING];
|
const char * json = "[1]";
|
||||||
strcpy(json, "[1]");
|
ondemand::document doc = parser.iterate(json, strlen(json));
|
||||||
ondemand::document doc = parser.iterate(json, strlen(json), sizeof(json));
|
|
||||||
```
|
```
|
||||||
|
|
||||||
We recommend against creating many `std::string` or many `std::padding_string` instances in your application to store your JSON data.
|
|
||||||
Consider reusing the same buffers and limiting memory allocations.
|
Consider reusing the same buffers and limiting memory allocations.
|
||||||
|
|
||||||
Documents Are Iterators
|
Documents Are Iterators
|
||||||
@@ -242,7 +239,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
|||||||
> parsing and writing out the unescaped keys to a string buffer and returning a `std::string_view`
|
> parsing and writing out the unescaped keys to a string buffer and returning a `std::string_view`
|
||||||
> instance. You should expect a performance penalty when using `unescaped_key()`.
|
> instance. You should expect a performance penalty when using `unescaped_key()`.
|
||||||
> ```c++
|
> ```c++
|
||||||
> auto json = R"({"k\u0065y": 1})"_padded;
|
> std::string json = R"({"k\u0065y": 1})";
|
||||||
> ondemand::parser parser;
|
> ondemand::parser parser;
|
||||||
> auto doc = parser.iterate(json);
|
> auto doc = parser.iterate(json);
|
||||||
> ondemand::object object = doc.get_object();
|
> ondemand::object object = doc.get_object();
|
||||||
@@ -264,7 +261,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
|||||||
>
|
>
|
||||||
> ```c++
|
> ```c++
|
||||||
> ondemand::parser parser;
|
> ondemand::parser parser;
|
||||||
> auto json = R"( { "x": 1, "y": 2 } )"_padded;
|
> std::string json = R"( { "x": 1, "y": 2 } )";
|
||||||
> auto doc = parser.iterate(json);
|
> auto doc = parser.iterate(json);
|
||||||
> double y = doc.find_field("y"); // The cursor is now after the 2 (at })
|
> double y = doc.find_field("y"); // The cursor is now after the 2 (at })
|
||||||
> double x = doc.find_field("x"); // This fails, because there are no more fields after "y"
|
> double x = doc.find_field("x"); // This fails, because there are no more fields after "y"
|
||||||
@@ -274,7 +271,7 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
|||||||
>
|
>
|
||||||
> ```c++
|
> ```c++
|
||||||
> ondemand::parser parser;
|
> ondemand::parser parser;
|
||||||
> auto json = R"( { "x": 1, "y": 2 } )"_padded;
|
> std::string json = R"( { "x": 1, "y": 2 } )";
|
||||||
> auto doc = parser.iterate(json);
|
> auto doc = parser.iterate(json);
|
||||||
> double y = doc["y"]; // The cursor is now after the 2 (at })
|
> double y = doc["y"]; // The cursor is now after the 2 (at })
|
||||||
> double x = doc["x"]; // Success: [] loops back around to find "x"
|
> double x = doc["x"]; // Success: [] loops back around to find "x"
|
||||||
@@ -291,23 +288,23 @@ support for users who avoid exceptions. See [the simdjson error handling documen
|
|||||||
* **Output to strings (simdjson 1.0 or better):** Given a document, a value, an array or an object in a JSON document, you can output a JSON string version suitable to be parsed again as JSON content: `simdjson::to_json_string(element)`. A call to `to_json_string` consumes fully the element: if you apply it on a document, the JSON pointer is advanced to the end of the document. The `simdjson::to_json_string` does not allocate memory. The `to_json_string` function should not be confused with retrieving the value of a string instance which are escaped and represented using a lightweight `std::string_view` instance pointing at an internal string buffer inside the parser instance. To illustrate, the first of the following two code segments will print the unescaped string `"test"` complete with the quote whereas the second one will print the escaped content of the string (without the quotes).
|
* **Output to strings (simdjson 1.0 or better):** Given a document, a value, an array or an object in a JSON document, you can output a JSON string version suitable to be parsed again as JSON content: `simdjson::to_json_string(element)`. A call to `to_json_string` consumes fully the element: if you apply it on a document, the JSON pointer is advanced to the end of the document. The `simdjson::to_json_string` does not allocate memory. The `to_json_string` function should not be confused with retrieving the value of a string instance which are escaped and represented using a lightweight `std::string_view` instance pointing at an internal string buffer inside the parser instance. To illustrate, the first of the following two code segments will print the unescaped string `"test"` complete with the quote whereas the second one will print the escaped content of the string (without the quotes).
|
||||||
> ```C++
|
> ```C++
|
||||||
> // serialize a JSON to an escaped std::string instance so that it can be parsed again as JSON
|
> // serialize a JSON to an escaped std::string instance so that it can be parsed again as JSON
|
||||||
> auto silly_json = R"( { "test": "result" } )"_padded;
|
> std::string silly_json = R"( { "test": "result" } )";
|
||||||
> ondemand::document doc = parser.iterate(silly_json);
|
> ondemand::document doc = parser.iterate(silly_json);
|
||||||
> std::cout << simdjson::to_json_string(doc["test"]) << std::endl; // Requires simdjson 1.0 or better
|
> std::cout << simdjson::to_json_string(doc["test"]) << std::endl; // Requires simdjson 1.0 or better
|
||||||
>````
|
>````
|
||||||
> ```C++
|
> ```C++
|
||||||
> // retrieves an unescaped string value as a string_view instance
|
> // retrieves an unescaped string value as a string_view instance
|
||||||
> auto silly_json = R"( { "test": "result" } )"_padded;
|
> std::string silly_json = R"( { "test": "result" } )";
|
||||||
> ondemand::document doc = parser.iterate(silly_json);
|
> ondemand::document doc = parser.iterate(silly_json);
|
||||||
> std::cout << std::string_view(doc["test"]) << std::endl;
|
> std::cout << std::string_view(doc["test"]) << std::endl;
|
||||||
>````
|
>````
|
||||||
You can use `to_json_string` to efficiently extract components of a JSON document to reconstruct a new JSON document, as in the following example:
|
You can use `to_json_string` to efficiently extract components of a JSON document to reconstruct a new JSON document, as in the following example:
|
||||||
> ```C++
|
> ```C++
|
||||||
> auto cars_json = R"( [
|
> std::string cars_json = R"( [
|
||||||
> { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
> { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
> { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
> { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
> { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
> { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
> ] )"_padded;
|
> ] )";
|
||||||
> std::vector<std::string_view> arrays;
|
> std::vector<std::string_view> arrays;
|
||||||
> // We are going to collect string_view instances which point inside the `cars_json` string
|
> // We are going to collect string_view instances which point inside the `cars_json` string
|
||||||
> // and are therefore valid as long as `cars_json` remains in scope.
|
> // and are therefore valid as long as `cars_json` remains in scope.
|
||||||
@@ -338,11 +335,11 @@ The following code illustrates many of the above concepts:
|
|||||||
|
|
||||||
```c++
|
```c++
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
// Iterating through an array of objects
|
// Iterating through an array of objects
|
||||||
for (ondemand::object car : parser.iterate(cars_json)) {
|
for (ondemand::object car : parser.iterate(cars_json)) {
|
||||||
@@ -366,10 +363,10 @@ Here is a different example illustrating the same ideas:
|
|||||||
|
|
||||||
```C++
|
```C++
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto points_json = R"( [
|
std::string points_json = R"( [
|
||||||
{ "12345" : {"x":12.34, "y":56.78, "z": 9998877} },
|
{ "12345" : {"x":12.34, "y":56.78, "z": 9998877} },
|
||||||
{ "12545" : {"x":11.44, "y":12.78, "z": 11111111} }
|
{ "12545" : {"x":11.44, "y":12.78, "z": 11111111} }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
// Parse and iterate through an array of objects
|
// Parse and iterate through an array of objects
|
||||||
for (ondemand::object points : parser.iterate(points_json)) {
|
for (ondemand::object points : parser.iterate(points_json)) {
|
||||||
@@ -385,9 +382,9 @@ for (ondemand::object points : parser.iterate(points_json)) {
|
|||||||
And another one:
|
And another one:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"(
|
std::string abstract_json = R"(
|
||||||
{ "str" : { "123" : {"abc" : 3.14 } } }
|
{ "str" : { "123" : {"abc" : 3.14 } } }
|
||||||
)"_padded;
|
)";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(abstract_json);
|
auto doc = parser.iterate(abstract_json);
|
||||||
cout << doc["str"]["123"]["abc"].get_double() << endl; // Prints 3.14
|
cout << doc["str"]["123"]["abc"].get_double() << endl; // Prints 3.14
|
||||||
@@ -399,9 +396,9 @@ cout << doc["str"]["123"]["abc"].get_double() << endl; // Prints 3.14
|
|||||||
to `get()` which gives you back an error code: e.g.,
|
to `get()` which gives you back an error code: e.g.,
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto abstract_json = R"(
|
std::string abstract_json = R"(
|
||||||
{ "str" : { "123" : {"abc" : 3.14 } } }
|
{ "str" : { "123" : {"abc" : 3.14 } } }
|
||||||
)"_padded;
|
)";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
|
|
||||||
double value;
|
double value;
|
||||||
@@ -417,7 +414,7 @@ aware that the `count_elements` method can be costly since it requires scanning
|
|||||||
whole array. You may use it as follows if your document is itself an array:
|
whole array. You may use it as follows if your document is itself an array:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto cars_json = R"( [ 40.1, 39.9, 37.7, 40.4 ] )"_padded;
|
std::string cars_json = R"( [ 40.1, 39.9, 37.7, 40.4 ] )";
|
||||||
auto doc = parser.iterate(cars_json);
|
auto doc = parser.iterate(cars_json);
|
||||||
size_t count = doc.count_elements(); // requires simdjson 1.0 or better
|
size_t count = doc.count_elements(); // requires simdjson 1.0 or better
|
||||||
std::vector<double> values(count);
|
std::vector<double> values(count);
|
||||||
@@ -429,7 +426,7 @@ If you access an array inside a document, you can use the `count_elements` metho
|
|||||||
You should not let the array instance go out of scope before consuming it after calling the `count_elements` method:
|
You should not let the array instance go out of scope before consuming it after calling the `count_elements` method:
|
||||||
``` C++
|
``` C++
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto cars_json = R"( { "test":[ { "val1":1, "val2":2 }, { "val1":1, "val2":2 } ] } )"_padded;
|
std::string cars_json = R"( { "test":[ { "val1":1, "val2":2 }, { "val1":1, "val2":2 } ] } )";
|
||||||
auto doc = parser.iterate(cars_json);
|
auto doc = parser.iterate(cars_json);
|
||||||
auto test_array = doc.find_field("test").get_array();
|
auto test_array = doc.find_field("test").get_array();
|
||||||
size_t count = test_array.count_elements(); // requires simdjson 1.0 or better
|
size_t count = test_array.count_elements(); // requires simdjson 1.0 or better
|
||||||
@@ -511,7 +508,7 @@ C++17 Support
|
|||||||
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
|
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
std::string json = R"( { "foo": 1, "bar": 2 } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::object object;
|
dom::object object;
|
||||||
auto error = parser.parse(json).get(object);
|
auto error = parser.parse(json).get(object);
|
||||||
@@ -525,7 +522,7 @@ For comparison, here is the C++ 11 version of the same code:
|
|||||||
|
|
||||||
```c++
|
```c++
|
||||||
// C++ 11 version for comparison
|
// C++ 11 version for comparison
|
||||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
std::string json = R"( { "foo": 1, "bar": 2 } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::object object;
|
dom::object object;
|
||||||
auto error = parser.parse(json).get(object);
|
auto error = parser.parse(json).get(object);
|
||||||
@@ -580,11 +577,11 @@ The simdjson library also supports [JSON pointer](https://tools.ietf.org/html/rf
|
|||||||
Consider the following example:
|
Consider the following example:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto cars = parser.iterate(cars_json);
|
auto cars = parser.iterate(cars_json);
|
||||||
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
||||||
@@ -598,11 +595,11 @@ select the value. If your keys contain the characters '/' or '~', they must be e
|
|||||||
For multiple JSON pointer queries on a document, one can call `at_pointer` multiple times.
|
For multiple JSON pointer queries on a document, one can call `at_pointer` multiple times.
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto cars = parser.iterate(cars_json);
|
auto cars = parser.iterate(cars_json);
|
||||||
size_t size = cars.count_elements();
|
size_t size = cars.count_elements();
|
||||||
@@ -627,11 +624,11 @@ struct car_type {
|
|||||||
make{_make}, model{_model}, year(_year), tire_pressure(_tire_pressure) {}
|
make{_make}, model{_model}, year(_year), tire_pressure(_tire_pressure) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document cars;
|
ondemand::document cars;
|
||||||
@@ -667,11 +664,11 @@ for (int i = 0; i < 3; i++) {
|
|||||||
Furthermore, `at_pointer` calls `rewind` at the beginning of the call (i.e. the document is not reset after `at_pointer`). Consider the following example,
|
Furthermore, `at_pointer` calls `rewind` at the beginning of the call (i.e. the document is not reset after `at_pointer`). Consider the following example,
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto json = R"( {
|
std::string json = R"( {
|
||||||
"k0": 27,
|
"k0": 27,
|
||||||
"k1": [13,26],
|
"k1": [13,26],
|
||||||
"k2": true
|
"k2": true
|
||||||
} )"_padded;
|
} )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
std::cout << doc.at_pointer("/k1/1") << std::endl; // Prints 26
|
std::cout << doc.at_pointer("/k1/1") << std::endl; // Prints 26
|
||||||
@@ -766,11 +763,11 @@ int main(void) {
|
|||||||
This is how the example in "Using the Parsed JSON" could be written using only error code checking:
|
This is how the example in "Using the Parsed JSON" could be written using only error code checking:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::array cars;
|
dom::array cars;
|
||||||
auto error = parser.parse(cars_json).get(cars);
|
auto error = parser.parse(cars_json).get(cars);
|
||||||
@@ -813,10 +810,10 @@ for (dom::element car_element : cars) {
|
|||||||
Here is another example:
|
Here is another example:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"( [
|
std::string abstract_json = R"( [
|
||||||
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
||||||
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::array array;
|
dom::array array;
|
||||||
auto error = parser.parse(abstract_json).get(array);
|
auto error = parser.parse(abstract_json).get(array);
|
||||||
@@ -846,8 +843,8 @@ for (dom::element elem : array) {
|
|||||||
And another one:
|
And another one:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"(
|
std::string abstract_json = R"(
|
||||||
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
|
{ "str" : { "123" : {"abc" : 3.14 } } } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
double v;
|
double v;
|
||||||
auto error = parser.parse(abstract_json)["str"]["123"]["abc"].get(v);
|
auto error = parser.parse(abstract_json)["str"]["123"]["abc"].get(v);
|
||||||
@@ -926,11 +923,11 @@ before printout the data.
|
|||||||
|
|
||||||
```C++
|
```C++
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
auto doc = parser.iterate(cars_json);
|
auto doc = parser.iterate(cars_json);
|
||||||
for (simdjson_unused ondemand::object car : doc) {
|
for (simdjson_unused ondemand::object car : doc) {
|
||||||
@@ -957,7 +954,7 @@ parse as you see fit.
|
|||||||
|
|
||||||
```C++
|
```C++
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"({"value":12321323213213213213213213213211223})"_padded;
|
std::string docdata = R"({"value":12321323213213213213213213213211223})";
|
||||||
simdjson::ondemand::document doc = parser.iterate(docdata);
|
simdjson::ondemand::document doc = parser.iterate(docdata);
|
||||||
simdjson::ondemand::object obj = doc.get_object();
|
simdjson::ondemand::object obj = doc.get_object();
|
||||||
std::string_view token = obj["value"].raw_json_token();
|
std::string_view token = obj["value"].raw_json_token();
|
||||||
@@ -970,7 +967,7 @@ source document.
|
|||||||
|
|
||||||
```C++
|
```C++
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"({"value":"12321323213213213213213213213211223"})"_padded;
|
std::string docdata = R"({"value":"12321323213213213213213213213211223"})";
|
||||||
simdjson::ondemand::document doc = parser.iterate(docdata);
|
simdjson::ondemand::document doc = parser.iterate(docdata);
|
||||||
simdjson::ondemand::object obj = doc.get_object();
|
simdjson::ondemand::object obj = doc.get_object();
|
||||||
string_view token = obj["value"].raw_json_token();
|
string_view token = obj["value"].raw_json_token();
|
||||||
@@ -993,7 +990,7 @@ than 4GB), though each individual document must be no larger than 4 GB.
|
|||||||
Here is a simple example:
|
Here is a simple example:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded;
|
std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream docs = parser.iterate_many(json);
|
ondemand::document_stream docs = parser.iterate_many(json);
|
||||||
for (auto & doc : docs) {
|
for (auto & doc : docs) {
|
||||||
@@ -1007,12 +1004,12 @@ It is important to note that the iteration returns a `document` reference, and h
|
|||||||
Unlike `parser.iterate`, `parser.iterate_many` may parse "on demand" (lazily). That is, no parsing may have been done before you enter the loop
|
Unlike `parser.iterate`, `parser.iterate_many` may parse "on demand" (lazily). That is, no parsing may have been done before you enter the loop
|
||||||
`for (auto & doc : docs) {` and you should expect the parser to only ever fully parse one JSON document at a time.
|
`for (auto & doc : docs) {` and you should expect the parser to only ever fully parse one JSON document at a time.
|
||||||
|
|
||||||
As with `parser.iterate`, when calling `parser.iterate_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.iterate_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents.
|
As with `parser.iterate`, when calling `parser.iterate_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.iterate_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]")`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents.
|
||||||
|
|
||||||
|
|
||||||
`iterate_many` can also take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents.
|
The `iterate_many` function can also take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents.
|
||||||
|
|
||||||
If your documents are large (e.g., larger than a megabyte), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`.
|
If your individual documents within the stream of documents are large (e.g., larger than a megabyte per individual document), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`.
|
||||||
|
|
||||||
See [iterate_many.md](iterate_many.md) for detailed information and design.
|
See [iterate_many.md](iterate_many.md) for detailed information and design.
|
||||||
|
|
||||||
|
|||||||
+26
-51
@@ -15,7 +15,6 @@ An overview of what you need to know to use simdjson, with examples.
|
|||||||
* [Reusing the parser for maximum efficiency](#reusing-the-parser-for-maximum-efficiency)
|
* [Reusing the parser for maximum efficiency](#reusing-the-parser-for-maximum-efficiency)
|
||||||
* [Server Loops: Long-Running Processes and Memory Capacity](#server-loops-long-running-processes-and-memory-capacity)
|
* [Server Loops: Long-Running Processes and Memory Capacity](#server-loops-long-running-processes-and-memory-capacity)
|
||||||
* [Best Use of the DOM API](#best-use-of-the-dom-api)
|
* [Best Use of the DOM API](#best-use-of-the-dom-api)
|
||||||
* [Padding and Temporary Copies](#padding-and-temporary-copies)
|
|
||||||
|
|
||||||
DOM vs On Demand
|
DOM vs On Demand
|
||||||
----------------------------------------------
|
----------------------------------------------
|
||||||
@@ -36,12 +35,11 @@ dom::parser parser;
|
|||||||
dom::element doc = parser.load(filename); // load and parse a file
|
dom::element doc = parser.load(filename); // load and parse a file
|
||||||
```
|
```
|
||||||
|
|
||||||
Or by creating a padded string (for efficiency reasons, simdjson requires a string with
|
Or by creating a string and calling `parse()`:
|
||||||
SIMDJSON_PADDING bytes at the end) and calling `parse()`:
|
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::element doc = parser.parse("[1,2,3]"_padded); // parse a string, the _padded suffix creates a simdjson::padded_string instance
|
dom::element doc = parser.parse(std::string("[1,2,3]")); // parse a string
|
||||||
```
|
```
|
||||||
|
|
||||||
The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
|
The parsed document resulting from the `parser.load` and `parser.parse` calls depends on the `parser` instance. Thus the `parser` instance must remain in scope. Furthermore, you must have at most one parsed document in play per `parser` instance.
|
||||||
@@ -54,8 +52,7 @@ During the`load` or `parse` calls, neither the input file nor the input string a
|
|||||||
For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson. [See our performance notes for details](performance.md).
|
For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson. [See our performance notes for details](performance.md).
|
||||||
|
|
||||||
If you need a lower-level interface, you may call the function `parser.parse(const char * p, size_t l)` on a pointer `p` while specifying the
|
If you need a lower-level interface, you may call the function `parser.parse(const char * p, size_t l)` on a pointer `p` while specifying the
|
||||||
length of your input `l` in bytes. To see how to get the very best performance from a low-level approach, you way want to read our [performance notes](https://github.com/simdjson/simdjson/blob/master/doc/performance.md#padding-and-temporary-copies) on this topic (see the Padding and Temporary Copies section).
|
length of your input `l` in bytes.
|
||||||
|
|
||||||
|
|
||||||
Using the Parsed JSON
|
Using the Parsed JSON
|
||||||
---------------------
|
---------------------
|
||||||
@@ -69,7 +66,7 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
|
|||||||
`dom::object` and `dom::array`) and pass it by reference to `get()` which gives you back an error code: e.g.,
|
`dom::object` and `dom::array`) and pass it by reference to `get()` which gives you back an error code: e.g.,
|
||||||
```c++
|
```c++
|
||||||
simdjson::error_code error;
|
simdjson::error_code error;
|
||||||
simdjson::padded_string numberstring = "1.2"_padded; // our JSON input ("1.2")
|
std::string numberstring = "1.2"; // our JSON input ("1.2")
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
double value; // variable where we store the value to be parsed
|
double value; // variable where we store the value to be parsed
|
||||||
error = parser.parse(numberstring).get(value);
|
error = parser.parse(numberstring).get(value);
|
||||||
@@ -96,11 +93,11 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
|
|||||||
The following code illustrates all of the above:
|
The following code illustrates all of the above:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
// Iterating through an array of objects
|
// Iterating through an array of objects
|
||||||
@@ -129,10 +126,10 @@ for (dom::object car : parser.parse(cars_json)) {
|
|||||||
Here is a different example illustrating the same ideas:
|
Here is a different example illustrating the same ideas:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"( [
|
std::string abstract_json = R"( [
|
||||||
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
||||||
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
// Parse and iterate through an array of objects
|
// Parse and iterate through an array of objects
|
||||||
@@ -151,8 +148,8 @@ And another one:
|
|||||||
|
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"(
|
std::string abstract_json = R"(
|
||||||
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
|
{ "str" : { "123" : {"abc" : 3.14 } } } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
|
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
|
||||||
cout << "number: " << v << endl;
|
cout << "number: " << v << endl;
|
||||||
@@ -165,7 +162,7 @@ C++17 Support
|
|||||||
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
|
While the simdjson library can be used in any project using C++ 11 and above, field iteration has special support C++ 17's destructuring syntax. For example:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
std::string json = R"( { "foo": 1, "bar": 2 } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::object object;
|
dom::object object;
|
||||||
auto error = parser.parse(json).get(object);
|
auto error = parser.parse(json).get(object);
|
||||||
@@ -179,7 +176,7 @@ For comparison, here is the C++ 11 version of the same code:
|
|||||||
|
|
||||||
```c++
|
```c++
|
||||||
// C++ 11 version for comparison
|
// C++ 11 version for comparison
|
||||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
std::string json = R"( { "foo": 1, "bar": 2 } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::object object;
|
dom::object object;
|
||||||
auto error = parser.parse(json).get(object);
|
auto error = parser.parse(json).get(object);
|
||||||
@@ -197,11 +194,11 @@ The simdjson library also supports [JSON pointer](https://tools.ietf.org/html/rf
|
|||||||
`at_pointer()` method, letting you reach further down into the document in a single call:
|
`at_pointer()` method, letting you reach further down into the document in a single call:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::element cars = parser.parse(cars_json);
|
dom::element cars = parser.parse(cars_json);
|
||||||
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
||||||
@@ -218,11 +215,11 @@ You can apply a JSON path to any node and the path gets interpreted relatively,
|
|||||||
Consider the following example:
|
Consider the following example:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::element cars = parser.parse(cars_json);
|
dom::element cars = parser.parse(cars_json);
|
||||||
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
||||||
@@ -321,11 +318,11 @@ int main(void) {
|
|||||||
This is how the example in "Using the Parsed JSON" could be written using only error code checking:
|
This is how the example in "Using the Parsed JSON" could be written using only error code checking:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::array cars;
|
dom::array cars;
|
||||||
auto error = parser.parse(cars_json).get(cars);
|
auto error = parser.parse(cars_json).get(cars);
|
||||||
@@ -368,10 +365,10 @@ for (dom::element car_element : cars) {
|
|||||||
Here is another example:
|
Here is another example:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"( [
|
std::string abstract_json = R"( [
|
||||||
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
||||||
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::array array;
|
dom::array array;
|
||||||
auto error = parser.parse(abstract_json).get(array);
|
auto error = parser.parse(abstract_json).get(array);
|
||||||
@@ -401,8 +398,8 @@ for (dom::element elem : array) {
|
|||||||
And another one:
|
And another one:
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto abstract_json = R"(
|
std::string abstract_json = R"(
|
||||||
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
|
{ "str" : { "123" : {"abc" : 3.14 } } } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
double v;
|
double v;
|
||||||
auto error = parser.parse(abstract_json)["str"]["123"]["abc"].get(v);
|
auto error = parser.parse(abstract_json)["str"]["123"]["abc"].get(v);
|
||||||
@@ -536,15 +533,16 @@ you can parse terabytes of JSON data without doing any new allocation.
|
|||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
// This initializes buffers and a document big enough to handle this JSON.
|
// This initializes buffers and a document big enough to handle this JSON.
|
||||||
dom::element doc = parser.parse("[ true, false ]"_padded);
|
dom::element doc = parser.parse(std::string("[ true, false ]"));
|
||||||
|
// std::string("[ true, false ]") may be replaced by "[ true, false ]"s
|
||||||
cout << doc << endl;
|
cout << doc << endl;
|
||||||
|
|
||||||
// This reuses the existing buffers, and reuses and *overwrites* the old document
|
// This reuses the existing buffers, and reuses and *overwrites* the old document
|
||||||
doc = parser.parse("[1, 2, 3]"_padded);
|
doc = parser.parse(std::string("[1, 2, 3]"));
|
||||||
cout << doc << endl;
|
cout << doc << endl;
|
||||||
|
|
||||||
// This also reuses the existing buffers, and reuses and *overwrites* the old document
|
// This also reuses the existing buffers, and reuses and *overwrites* the old document
|
||||||
dom::element doc2 = parser.parse("true"_padded);
|
dom::element doc2 = parser.parse(std::string("true"));
|
||||||
// Even if you keep the old reference around, doc and doc2 refer to the same document.
|
// Even if you keep the old reference around, doc and doc2 refer to the same document.
|
||||||
cout << doc << endl;
|
cout << doc << endl;
|
||||||
cout << doc2 << endl;
|
cout << doc2 << endl;
|
||||||
@@ -610,26 +608,3 @@ Best Use of the DOM API
|
|||||||
-------------------------
|
-------------------------
|
||||||
|
|
||||||
The simdjson API provides access to the JSON DOM (document-object-model) content as a tree of `dom::element` instances, each representing an object, an array or an atomic type (null, true, false, number). These `dom::element` instances are lightweight objects (e.g., spanning 16 bytes) and it might be advantageous to pass them by value, as opposed to passing them by reference or by pointer.
|
The simdjson API provides access to the JSON DOM (document-object-model) content as a tree of `dom::element` instances, each representing an object, an array or an atomic type (null, true, false, number). These `dom::element` instances are lightweight objects (e.g., spanning 16 bytes) and it might be advantageous to pass them by value, as opposed to passing them by reference or by pointer.
|
||||||
|
|
||||||
Padding and Temporary Copies
|
|
||||||
--------------
|
|
||||||
|
|
||||||
The simdjson function `parser.parse` reads data from a padded buffer, containing SIMDJSON_PADDING extra bytes added at the end.
|
|
||||||
If you are passing a `padded_string` to `parser.parse` or loading the JSON directly from
|
|
||||||
disk (`parser.load`), padding is automatically handled.
|
|
||||||
When calling `parser.parse` on a pointer (e.g., `parser.parse(my_char_pointer, my_length_in_bytes)`) a temporary copy is made by default with adequate padding and you, again, do not need to be concerned with padding.
|
|
||||||
|
|
||||||
Some users may not be able use our `padded_string` class or to load the data directly from disk (`parser.load`). They may need to pass data pointers to the library. If these users wish to avoid temporary copies and corresponding temporary memory allocations, they may want to call `parser.parse` with the `realloc_if_needed` parameter set to false (e.g., `parser.parse(my_char_pointer, my_length_in_bytes, false)`). In such cases, they need to ensure that there are at least SIMDJSON_PADDING extra bytes at the end that can be safely accessed and read. They do not need to initialize the padded bytes to any value in particular. The following example is safe:
|
|
||||||
|
|
||||||
|
|
||||||
```C++
|
|
||||||
const char *json = R"({"key":"value"})";
|
|
||||||
const size_t json_len = std::strlen(json);
|
|
||||||
std::unique_ptr<char[]> padded_json_copy{new char[json_len + SIMDJSON_PADDING]};
|
|
||||||
memcpy(padded_json_copy.get(), json, json_len);
|
|
||||||
memset(padded_json_copy.get() + json_len, 0, SIMDJSON_PADDING);
|
|
||||||
simdjson::dom::parser parser;
|
|
||||||
simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false);
|
|
||||||
````
|
|
||||||
|
|
||||||
Setting the `realloc_if_needed` parameter `false` in this manner may lead to better performance since copies are avoided, but it requires that the user takes more responsibilities: the simdjson library cannot verify that the input buffer was padded with SIMDJSON_PADDING extra bytes.
|
|
||||||
|
|||||||
+2
-2
@@ -167,7 +167,7 @@ Let us illustrate the idea with code:
|
|||||||
|
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )";
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::ondemand::document_stream stream;
|
simdjson::ondemand::document_stream stream;
|
||||||
auto error = parser.iterate_many(json).get(stream);
|
auto error = parser.iterate_many(json).get(stream);
|
||||||
@@ -208,7 +208,7 @@ Some users may need to work with truncated streams. The simdjson may truncate do
|
|||||||
Consider the following example where a truncated document (`{"key":"intentionally unclosed string `) containing 39 bytes has been left within the stream. In such cases, the first two whole documents are parsed and returned, and the `truncated_bytes()` method returns 39.
|
Consider the following example where a truncated document (`{"key":"intentionally unclosed string `) containing 39 bytes has been left within the stream. In such cases, the first two whole documents are parsed and returned, and the `truncated_bytes()` method returns 39.
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )";
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::ondemand::document_stream stream;
|
simdjson::ondemand::document_stream stream;
|
||||||
auto error = parser.iterate_many(json,json.size()).get(stream);
|
auto error = parser.iterate_many(json,json.size()).get(stream);
|
||||||
|
|||||||
+2
-2
@@ -174,7 +174,7 @@ Let us illustrate the idea with code:
|
|||||||
|
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
auto error = parser.parse_many(json).get(stream);
|
auto error = parser.parse_many(json).get(stream);
|
||||||
@@ -215,7 +215,7 @@ Some users may need to work with truncated streams. The simdjson may truncate do
|
|||||||
Consider the following example where a truncated document (`{"key":"intentionally unclosed string `) containing 39 bytes has been left within the stream. In such cases, the first two whole documents are parsed and returned, and the `truncated_bytes()` method returns 39.
|
Consider the following example where a truncated document (`{"key":"intentionally unclosed string `) containing 39 bytes has been left within the stream. In such cases, the first two whole documents are parsed and returned, and the `truncated_bytes()` method returns 39.
|
||||||
|
|
||||||
```C++
|
```C++
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
auto error = parser.parse_many(json,json.size()).get(stream);
|
auto error = parser.parse_many(json,json.size()).get(stream);
|
||||||
|
|||||||
+3
-9
@@ -24,14 +24,14 @@ you can parse terabytes of JSON data without doing any new allocation.
|
|||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
|
|
||||||
// This initializes buffers big enough to handle this JSON.
|
// This initializes buffers big enough to handle this JSON.
|
||||||
auto json = "[ true, false ]"_padded;
|
std::string json = "[ true, false ]";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
for(bool i : doc.get_array()) {
|
for(bool i : doc.get_array()) {
|
||||||
cout << i << endl;
|
cout << i << endl;
|
||||||
}
|
}
|
||||||
|
|
||||||
// This reuses the existing buffers
|
// This reuses the existing buffers
|
||||||
auto number_json = "[1, 2, 3]"_padded;
|
std::string number_json = "[1, 2, 3]";
|
||||||
doc = parser.iterate(number_json);
|
doc = parser.iterate(number_json);
|
||||||
for(int64_t i : doc.get_array()) {
|
for(int64_t i : doc.get_array()) {
|
||||||
cout << i << endl;
|
cout << i << endl;
|
||||||
@@ -45,15 +45,9 @@ Reusing string buffers
|
|||||||
We recommend against creating many `std::string` or `simdjson::padded_string` instances to store the JSON content in your application. [Creating many non-trivial objects is convenient but often surprisingly slow](https://lemire.me/blog/2020/08/08/performance-tip-constructing-many-non-trivial-objects-is-slow/). Instead, as much as possible, you should allocate (once or a few times) reusable memory buffers where you write your JSON content. If you have a buffer `json_str` (of type `char*`) allocated for `capacity` bytes and you store a JSON document spanning `length` bytes, you can pass it to simdjson as follows:
|
We recommend against creating many `std::string` or `simdjson::padded_string` instances to store the JSON content in your application. [Creating many non-trivial objects is convenient but often surprisingly slow](https://lemire.me/blog/2020/08/08/performance-tip-constructing-many-non-trivial-objects-is-slow/). Instead, as much as possible, you should allocate (once or a few times) reusable memory buffers where you write your JSON content. If you have a buffer `json_str` (of type `char*`) allocated for `capacity` bytes and you store a JSON document spanning `length` bytes, you can pass it to simdjson as follows:
|
||||||
|
|
||||||
```c++
|
```c++
|
||||||
auto doc = parser.iterate(padded_string_view(json_str, length, capacity));
|
auto doc = parser.iterate(json_str, length));
|
||||||
```
|
```
|
||||||
|
|
||||||
or simply
|
|
||||||
|
|
||||||
|
|
||||||
```c++
|
|
||||||
auto doc = parser.iterate(json_str, length, capacity);
|
|
||||||
```
|
|
||||||
|
|
||||||
|
|
||||||
Large files and huge page support
|
Large files and huge page support
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ namespace simdjson {
|
|||||||
namespace SIMDJSON_IMPLEMENTATION {
|
namespace SIMDJSON_IMPLEMENTATION {
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
// we don't have SSE, so let us use a scalar function
|
|
||||||
// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
// credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/
|
||||||
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||||
uint64_t val;
|
uint64_t val;
|
||||||
@@ -19,7 +18,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
|
|||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
} // namespace simdjson
|
} // namespace simdjson
|
||||||
|
|
||||||
#define SWAR_NUMBER_PARSING
|
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||||
|
|
||||||
#include "simdjson/generic/numberparsing.h"
|
#include "simdjson/generic/numberparsing.h"
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,10 @@ using namespace simd;
|
|||||||
struct backslash_and_quote {
|
struct backslash_and_quote {
|
||||||
public:
|
public:
|
||||||
static constexpr uint32_t BYTES_PROCESSED = 32;
|
static constexpr uint32_t BYTES_PROCESSED = 32;
|
||||||
simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
|
||||||
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst);
|
||||||
|
|
||||||
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
||||||
simdjson_really_inline bool has_backslash() { return bs_bits != 0; }
|
simdjson_really_inline bool has_backslash() { return bs_bits != 0; }
|
||||||
@@ -26,10 +29,21 @@ public:
|
|||||||
uint32_t quote_bits;
|
uint32_t quote_bits;
|
||||||
}; // struct backslash_and_quote
|
}; // struct backslash_and_quote
|
||||||
|
|
||||||
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
simdjson_really_inline backslash_and_quote
|
||||||
// this can read up to 31 bytes beyond the buffer size, but we require
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
|
||||||
// SIMDJSON_PADDING of padding
|
// If we don't have enough memory left to load a whole simd register, copy it first.
|
||||||
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
|
uint8_t tmpbuf[BYTES_PROCESSED];
|
||||||
|
if (simdjson_unlikely(src > last_full_buf)) {
|
||||||
|
SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
|
||||||
|
std::memset(tmpbuf, 0, BYTES_PROCESSED);
|
||||||
|
std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
|
||||||
|
src = tmpbuf;
|
||||||
|
}
|
||||||
|
return copy_and_find(src, dst);
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline backslash_and_quote
|
||||||
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
||||||
simd8<uint8_t> v0(src);
|
simd8<uint8_t> v0(src);
|
||||||
simd8<uint8_t> v1(src + sizeof(v0));
|
simd8<uint8_t> v1(src + sizeof(v0));
|
||||||
v0.store(dst);
|
v0.store(dst);
|
||||||
|
|||||||
@@ -35,12 +35,7 @@ double from_chars(const char *first, const char* end) noexcept;
|
|||||||
constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
|
constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The amount of padding needed in a buffer to parse JSON.
|
* Padding requirement.
|
||||||
*
|
|
||||||
* the input buf should be readable up to buf + SIMDJSON_PADDING
|
|
||||||
* this is a stopgap; there should be a better description of the
|
|
||||||
* main loop and its behavior that abstracts over this
|
|
||||||
* See https://github.com/simdjson/simdjson/issues/174
|
|
||||||
*/
|
*/
|
||||||
constexpr size_t SIMDJSON_PADDING = 32;
|
constexpr size_t SIMDJSON_PADDING = 32;
|
||||||
|
|
||||||
@@ -71,8 +66,12 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
|||||||
#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
|
#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
|
||||||
|
|
||||||
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO)
|
#if defined(SIMDJSON_REGULAR_VISUAL_STUDIO)
|
||||||
|
#if SIMDJSON_NO_FORCE_INLINING
|
||||||
|
// forcing inlining can increase stack usage.
|
||||||
|
#define simdjson_really_inline inline
|
||||||
|
#else
|
||||||
#define simdjson_really_inline __forceinline
|
#define simdjson_really_inline __forceinline
|
||||||
|
#endif
|
||||||
#define simdjson_never_inline __declspec(noinline)
|
#define simdjson_never_inline __declspec(noinline)
|
||||||
|
|
||||||
#define simdjson_unused
|
#define simdjson_unused
|
||||||
@@ -106,8 +105,12 @@ constexpr size_t DEFAULT_MAX_DEPTH = 1024;
|
|||||||
#define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop ))
|
#define SIMDJSON_POP_DISABLE_WARNINGS __pragma(warning( pop ))
|
||||||
|
|
||||||
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
|
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
|
||||||
|
#if SIMDJSON_NO_FORCE_INLINING
|
||||||
|
// forcing inlining can increase stack usage.
|
||||||
|
#define simdjson_really_inline inline
|
||||||
|
#else
|
||||||
#define simdjson_really_inline inline __attribute__((always_inline))
|
#define simdjson_really_inline inline __attribute__((always_inline))
|
||||||
|
#endif
|
||||||
#define simdjson_never_inline inline __attribute__((noinline))
|
#define simdjson_never_inline inline __attribute__((noinline))
|
||||||
|
|
||||||
#define simdjson_unused __attribute__((unused))
|
#define simdjson_unused __attribute__((unused))
|
||||||
@@ -253,10 +256,15 @@ namespace std {
|
|||||||
#endif
|
#endif
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// Feature flag for partially-implemented "don't require padding" feature
|
// The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding"
|
||||||
// TODO remove once feature complete.
|
// feature in the On Demand API.
|
||||||
#ifndef __SIMDJSON_CHECK_EOF
|
// When we have padding, we do not need to check for the end of the input buffer.
|
||||||
# define __SIMDJSON_CHECK_EOF 1
|
// However, without padding, it is unsafe not to have end-of-buffer checks.
|
||||||
|
// Thus this SIMDJSON_CHECK_EOF should be set to true (1) for safety as it activates
|
||||||
|
// several safety checks. We still allow expert users to disable it.
|
||||||
|
// Note that this only affects the On Demand API.
|
||||||
|
#ifndef SIMDJSON_CHECK_EOF
|
||||||
|
# define SIMDJSON_CHECK_EOF 1
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if SIMDJSON_CPLUSPLUS17
|
#if SIMDJSON_CPLUSPLUS17
|
||||||
@@ -268,11 +276,12 @@ namespace std {
|
|||||||
#if __has_attribute(__fallthrough__)
|
#if __has_attribute(__fallthrough__)
|
||||||
// we are good to go:
|
// we are good to go:
|
||||||
# define simdjson_fallthrough __attribute__((__fallthrough__))
|
# define simdjson_fallthrough __attribute__((__fallthrough__))
|
||||||
#endif
|
#endif // __has_attribute(__fallthrough__)
|
||||||
#endif
|
#endif // SIMDJSON_CPLUSPLUS17
|
||||||
|
|
||||||
// on some systems, we simply do not have support for fallthrough, so use a default:
|
// on some systems, we simply do not have support for fallthrough, so use a default:
|
||||||
#ifndef simdjson_fallthrough
|
#ifndef simdjson_fallthrough
|
||||||
# define simdjson_fallthrough do {} while (0) /* fallthrough */
|
# define simdjson_fallthrough do {} while (0) /* fallthrough */
|
||||||
#endif
|
#endif // simdjson_fallthrough
|
||||||
|
|
||||||
#endif // SIMDJSON_COMMON_DEFS_H
|
#endif // SIMDJSON_COMMON_DEFS_H
|
||||||
|
|||||||
@@ -100,7 +100,8 @@ inline simdjson_result<element> parser::parse_into_document(document& provided_d
|
|||||||
// Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!!
|
// Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!!
|
||||||
error_code _error = ensure_capacity(provided_doc, len);
|
error_code _error = ensure_capacity(provided_doc, len);
|
||||||
if (_error) { return _error; }
|
if (_error) { return _error; }
|
||||||
if (realloc_if_needed) {
|
(void)realloc_if_needed;
|
||||||
|
/*if (realloc_if_needed) {
|
||||||
// Make sure we have enough capacity to copy len bytes
|
// Make sure we have enough capacity to copy len bytes
|
||||||
if (!loaded_bytes || _loaded_bytes_capacity < len) {
|
if (!loaded_bytes || _loaded_bytes_capacity < len) {
|
||||||
loaded_bytes.reset( internal::allocate_padded_buffer(len) );
|
loaded_bytes.reset( internal::allocate_padded_buffer(len) );
|
||||||
@@ -112,6 +113,8 @@ inline simdjson_result<element> parser::parse_into_document(document& provided_d
|
|||||||
std::memcpy(static_cast<void *>(loaded_bytes.get()), buf, len);
|
std::memcpy(static_cast<void *>(loaded_bytes.get()), buf, len);
|
||||||
}
|
}
|
||||||
_error = implementation->parse(realloc_if_needed ? reinterpret_cast<const uint8_t*>(loaded_bytes.get()): buf, len, provided_doc);
|
_error = implementation->parse(realloc_if_needed ? reinterpret_cast<const uint8_t*>(loaded_bytes.get()): buf, len, provided_doc);
|
||||||
|
*/
|
||||||
|
_error = implementation->parse(buf, len, provided_doc);
|
||||||
|
|
||||||
if (_error) { return _error; }
|
if (_error) { return _error; }
|
||||||
|
|
||||||
@@ -122,8 +125,16 @@ simdjson_really_inline simdjson_result<element> parser::parse_into_document(docu
|
|||||||
return parse_into_document(provided_doc, reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
|
return parse_into_document(provided_doc, reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
|
||||||
}
|
}
|
||||||
simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept {
|
simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, const std::string &s) & noexcept {
|
||||||
return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
|
return parse_into_document(provided_doc, s.data(), s.length());
|
||||||
}
|
}
|
||||||
|
simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, std::string&& s) & noexcept {
|
||||||
|
std::string local_string(s);
|
||||||
|
return parse_into_document(provided_doc, local_string.data(), local_string.length());
|
||||||
|
}
|
||||||
|
simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, std::string_view s) & noexcept {
|
||||||
|
return parse_into_document(provided_doc, s.data(), s.length());
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept {
|
simdjson_really_inline simdjson_result<element> parser::parse_into_document(document& provided_doc, const padded_string &s) & noexcept {
|
||||||
return parse_into_document(provided_doc, s.data(), s.length(), false);
|
return parse_into_document(provided_doc, s.data(), s.length(), false);
|
||||||
}
|
}
|
||||||
@@ -137,7 +148,14 @@ simdjson_really_inline simdjson_result<element> parser::parse(const char *buf, s
|
|||||||
return parse(reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
|
return parse(reinterpret_cast<const uint8_t *>(buf), len, realloc_if_needed);
|
||||||
}
|
}
|
||||||
simdjson_really_inline simdjson_result<element> parser::parse(const std::string &s) & noexcept {
|
simdjson_really_inline simdjson_result<element> parser::parse(const std::string &s) & noexcept {
|
||||||
return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING);
|
return parse(s.data(), s.length());
|
||||||
|
}
|
||||||
|
simdjson_really_inline simdjson_result<element> parser::parse(std::string &&s) & noexcept {
|
||||||
|
std::string local_string(s);
|
||||||
|
return parse(local_string.data(), local_string.length());
|
||||||
|
}
|
||||||
|
simdjson_really_inline simdjson_result<element> parser::parse(std::string_view s) & noexcept {
|
||||||
|
return parse(s.data(), s.length());
|
||||||
}
|
}
|
||||||
simdjson_really_inline simdjson_result<element> parser::parse(const padded_string &s) & noexcept {
|
simdjson_really_inline simdjson_result<element> parser::parse(const padded_string &s) & noexcept {
|
||||||
return parse(s.data(), s.length(), false);
|
return parse(s.data(), s.length(), false);
|
||||||
@@ -153,6 +171,9 @@ inline simdjson_result<document_stream> parser::parse_many(const char *buf, size
|
|||||||
inline simdjson_result<document_stream> parser::parse_many(const std::string &s, size_t batch_size) noexcept {
|
inline simdjson_result<document_stream> parser::parse_many(const std::string &s, size_t batch_size) noexcept {
|
||||||
return parse_many(s.data(), s.length(), batch_size);
|
return parse_many(s.data(), s.length(), batch_size);
|
||||||
}
|
}
|
||||||
|
inline simdjson_result<document_stream> parser::parse_many(const std::string_view s, size_t batch_size) noexcept {
|
||||||
|
return parse_many(s.data(), s.length(), batch_size);
|
||||||
|
}
|
||||||
inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size) noexcept {
|
inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size) noexcept {
|
||||||
return parse_many(s.data(), s.length(), batch_size);
|
return parse_many(s.data(), s.length(), batch_size);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -142,63 +142,43 @@ public:
|
|||||||
* the parser instance without moving it by wrapping it inside an `unique_ptr` instance like
|
* the parser instance without moving it by wrapping it inside an `unique_ptr` instance like
|
||||||
* so: `std::unique_ptr<dom::parser> parser(new dom::parser{});`.
|
* so: `std::unique_ptr<dom::parser> parser(new dom::parser{});`.
|
||||||
*
|
*
|
||||||
* ### REQUIRED: Buffer Padding
|
* ### Buffer Padding
|
||||||
*
|
*
|
||||||
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
|
* We do not require that the input buffer be padded, but for backward compatibility, we have
|
||||||
* those bytes are initialized to, as long as they are allocated.
|
* a realloc_if_needed parameter with a default value. It can be ignored.
|
||||||
*
|
|
||||||
* If realloc_if_needed is true (the default), it is assumed that the buffer does *not* have enough padding,
|
|
||||||
* and it is copied into an enlarged temporary buffer before parsing. Thus the following is safe:
|
|
||||||
*
|
|
||||||
* const char *json = R"({"key":"value"})";
|
|
||||||
* const size_t json_len = std::strlen(json);
|
|
||||||
* simdjson::dom::parser parser;
|
|
||||||
* simdjson::dom::element element = parser.parse(json, json_len);
|
|
||||||
*
|
|
||||||
* If you set realloc_if_needed to false (e.g., parser.parse(json, json_len, false)),
|
|
||||||
* you must provide a buffer with at least SIMDJSON_PADDING extra bytes at the end.
|
|
||||||
* The benefit of setting realloc_if_needed to false is that you avoid a temporary
|
|
||||||
* memory allocation and a copy.
|
|
||||||
*
|
|
||||||
* The padded bytes may be read. It is not important how you initialize
|
|
||||||
* these bytes though we recommend a sensible default like null character values or spaces.
|
|
||||||
* For example, the following low-level code is safe:
|
|
||||||
*
|
|
||||||
* const char *json = R"({"key":"value"})";
|
|
||||||
* const size_t json_len = std::strlen(json);
|
|
||||||
* std::unique_ptr<char[]> padded_json_copy{new char[json_len + SIMDJSON_PADDING]};
|
|
||||||
* std::memcpy(padded_json_copy.get(), json, json_len);
|
|
||||||
* std::memset(padded_json_copy.get() + json_len, '\0', SIMDJSON_PADDING);
|
|
||||||
* simdjson::dom::parser parser;
|
|
||||||
* simdjson::dom::element element = parser.parse(padded_json_copy.get(), json_len, false);
|
|
||||||
*
|
*
|
||||||
* ### Parser Capacity
|
* ### Parser Capacity
|
||||||
*
|
*
|
||||||
* If the parser's current capacity is less than len, it will allocate enough capacity
|
* If the parser's current capacity is less than len, it will allocate enough capacity
|
||||||
* to handle it (up to max_capacity).
|
* to handle it (up to max_capacity).
|
||||||
*
|
*
|
||||||
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
|
* @param buf The JSON to parse.
|
||||||
* realloc_if_needed is true.
|
|
||||||
* @param len The length of the JSON.
|
* @param len The length of the JSON.
|
||||||
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
|
* @param realloc_if_needed this parameter is left in place for backward compatibility, it can be ignored (defaults to false)
|
||||||
* @return An element pointing at the root of the document, or an error:
|
* @return An element pointing at the root of the document, or an error:
|
||||||
* - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity,
|
* - MEMALLOC if the parser does not have enough capacity,
|
||||||
* and memory allocation fails.
|
* and memory allocation fails.
|
||||||
* - CAPACITY if the parser does not have enough capacity and len > max_capacity.
|
* - CAPACITY if the parser does not have enough capacity and len > max_capacity.
|
||||||
* - other json errors if parsing fails. You should not rely on these errors to always the same for the
|
* - other json errors if parsing fails. You should not rely on these errors to always the same for the
|
||||||
* same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
|
* same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
|
||||||
*/
|
*/
|
||||||
inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept;
|
inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = false) & noexcept;
|
||||||
inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete;
|
inline simdjson_result<element> parse(const uint8_t *buf, size_t len, bool realloc_if_needed = false) && =delete;
|
||||||
/** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
/** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
||||||
simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
|
simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = false) & noexcept;
|
||||||
simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
|
simdjson_really_inline simdjson_result<element> parse(const char *buf, size_t len, bool realloc_if_needed = false) && =delete;
|
||||||
/** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
/** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
||||||
|
simdjson_really_inline simdjson_result<element> parse(std::string_view s) & noexcept;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse(std::string_view s) && = delete;
|
||||||
simdjson_really_inline simdjson_result<element> parse(const std::string &s) & noexcept;
|
simdjson_really_inline simdjson_result<element> parse(const std::string &s) & noexcept;
|
||||||
simdjson_really_inline simdjson_result<element> parse(const std::string &s) && =delete;
|
simdjson_really_inline simdjson_result<element> parse(const std::string &s) && = delete;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse(std::string &&s) & noexcept;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse(std::string &&s) && = delete;
|
||||||
|
|
||||||
/** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
/** @overload parse(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
||||||
simdjson_really_inline simdjson_result<element> parse(const padded_string &s) & noexcept;
|
simdjson_really_inline simdjson_result<element> parse(const padded_string &s) & noexcept;
|
||||||
simdjson_really_inline simdjson_result<element> parse(const padded_string &s) && =delete;
|
simdjson_really_inline simdjson_result<element> parse(const padded_string &s) && =delete;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse(const padded_string &&s) = delete;
|
||||||
|
|
||||||
/** @private We do not want to allow implicit conversion from C string to std::string. */
|
/** @private We do not want to allow implicit conversion from C string to std::string. */
|
||||||
simdjson_really_inline simdjson_result<element> parse(const char *buf) noexcept = delete;
|
simdjson_really_inline simdjson_result<element> parse(const char *buf) noexcept = delete;
|
||||||
@@ -236,25 +216,28 @@ public:
|
|||||||
* moving a document, you can recover safe access to the document root with its `root()` method.
|
* moving a document, you can recover safe access to the document root with its `root()` method.
|
||||||
*
|
*
|
||||||
* @param doc The document instance where the parsed data will be stored (on success).
|
* @param doc The document instance where the parsed data will be stored (on success).
|
||||||
* @param buf The JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes, unless
|
* @param buf The JSON to parse.
|
||||||
* realloc_if_needed is true.
|
|
||||||
* @param len The length of the JSON.
|
* @param len The length of the JSON.
|
||||||
* @param realloc_if_needed Whether to reallocate and enlarge the JSON buffer to add padding.
|
* @param realloc_if_needed this parameter is left in place for backward compatibility, it can be ignored (defaults to false)
|
||||||
* @return An element pointing at the root of document, or an error:
|
* @return An element pointing at the root of document, or an error:
|
||||||
* - MEMALLOC if realloc_if_needed is true or the parser does not have enough capacity,
|
* - MEMALLOC if the parser does not have enough capacity,
|
||||||
* and memory allocation fails.
|
* and memory allocation fails.
|
||||||
* - CAPACITY if the parser does not have enough capacity and len > max_capacity.
|
* - CAPACITY if the parser does not have enough capacity and len > max_capacity.
|
||||||
* - other json errors if parsing fails. You should not rely on these errors to always the same for the
|
* - other json errors if parsing fails. You should not rely on these errors to always the same for the
|
||||||
* same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
|
* same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
|
||||||
*/
|
*/
|
||||||
inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) & noexcept;
|
inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = false) & noexcept;
|
||||||
inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = true) && =delete;
|
inline simdjson_result<element> parse_into_document(document& doc, const uint8_t *buf, size_t len, bool realloc_if_needed = false) && =delete;
|
||||||
/** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
/** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
||||||
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) & noexcept;
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = false) & noexcept;
|
||||||
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = true) && =delete;
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const char *buf, size_t len, bool realloc_if_needed = false) && =delete;
|
||||||
/** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
/** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
||||||
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const std::string &s) & noexcept;
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const std::string &s) & noexcept;
|
||||||
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const std::string &s) && =delete;
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const std::string &s) && =delete;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string &&s) & noexcept;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string &&s) && =delete;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string_view s) & noexcept;
|
||||||
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, std::string_view s) && =delete;
|
||||||
/** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
/** @overload parse_into_document(const uint8_t *buf, size_t len, bool realloc_if_needed) */
|
||||||
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const padded_string &s) & noexcept;
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const padded_string &s) & noexcept;
|
||||||
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const padded_string &s) && =delete;
|
simdjson_really_inline simdjson_result<element> parse_into_document(document& doc, const padded_string &s) && =delete;
|
||||||
@@ -402,10 +385,6 @@ public:
|
|||||||
* cout << title << endl;
|
* cout << title << endl;
|
||||||
* }
|
* }
|
||||||
*
|
*
|
||||||
* ### REQUIRED: Buffer Padding
|
|
||||||
*
|
|
||||||
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
|
|
||||||
* those bytes are initialized to, as long as they are allocated.
|
|
||||||
*
|
*
|
||||||
* ### Threads
|
* ### Threads
|
||||||
*
|
*
|
||||||
@@ -417,7 +396,7 @@ public:
|
|||||||
* If the parser's current capacity is less than batch_size, it will allocate enough capacity
|
* If the parser's current capacity is less than batch_size, it will allocate enough capacity
|
||||||
* to handle it (up to max_capacity).
|
* to handle it (up to max_capacity).
|
||||||
*
|
*
|
||||||
* @param buf The concatenated JSON to parse. Must have at least len + SIMDJSON_PADDING allocated bytes.
|
* @param buf The concatenated JSON to parse.
|
||||||
* @param len The length of the concatenated JSON.
|
* @param len The length of the concatenated JSON.
|
||||||
* @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
|
* @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
|
||||||
* spot is cache-related: small enough to fit in cache, yet big enough to
|
* spot is cache-related: small enough to fit in cache, yet big enough to
|
||||||
@@ -434,10 +413,11 @@ public:
|
|||||||
inline simdjson_result<document_stream> parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
inline simdjson_result<document_stream> parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
||||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
||||||
inline simdjson_result<document_stream> parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
inline simdjson_result<document_stream> parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
||||||
inline simdjson_result<document_stream> parse_many(const std::string &&s, size_t batch_size) = delete;// unsafe
|
inline simdjson_result<document_stream> parse_many(const std::string_view s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
||||||
|
inline simdjson_result<document_stream> parse_many(std::string &&s, size_t batch_size) = delete;// unsafe
|
||||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
||||||
inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept;
|
||||||
inline simdjson_result<document_stream> parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe
|
inline simdjson_result<document_stream> parse_many(padded_string &&s, size_t batch_size) = delete;// unsafe
|
||||||
|
|
||||||
/** @private We do not want to allow implicit conversion from C string to std::string. */
|
/** @private We do not want to allow implicit conversion from C string to std::string. */
|
||||||
simdjson_result<document_stream> parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete;
|
simdjson_result<document_stream> parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete;
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
|
|||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
} // namespace simdjson
|
} // namespace simdjson
|
||||||
|
|
||||||
#define SWAR_NUMBER_PARSING
|
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||||
#include "simdjson/generic/numberparsing.h"
|
#include "simdjson/generic/numberparsing.h"
|
||||||
|
|
||||||
#endif // SIMDJSON_FALLBACK_NUMBERPARSING_H
|
#endif // SIMDJSON_FALLBACK_NUMBERPARSING_H
|
||||||
|
|||||||
@@ -11,7 +11,10 @@ namespace {
|
|||||||
struct backslash_and_quote {
|
struct backslash_and_quote {
|
||||||
public:
|
public:
|
||||||
static constexpr uint32_t BYTES_PROCESSED = 1;
|
static constexpr uint32_t BYTES_PROCESSED = 1;
|
||||||
simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
|
||||||
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst);
|
||||||
|
|
||||||
simdjson_really_inline bool has_quote_first() { return c == '"'; }
|
simdjson_really_inline bool has_quote_first() { return c == '"'; }
|
||||||
simdjson_really_inline bool has_backslash() { return c == '\\'; }
|
simdjson_really_inline bool has_backslash() { return c == '\\'; }
|
||||||
@@ -21,6 +24,10 @@ public:
|
|||||||
uint8_t c;
|
uint8_t c;
|
||||||
}; // struct backslash_and_quote
|
}; // struct backslash_and_quote
|
||||||
|
|
||||||
|
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *) {
|
||||||
|
return copy_and_find(src, dst);
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
||||||
// store to dest unconditionally - we can overwrite the bits we don't like later
|
// store to dest unconditionally - we can overwrite the bits we don't like later
|
||||||
dst[0] = src[0];
|
dst[0] = src[0];
|
||||||
|
|||||||
@@ -15,47 +15,50 @@ simdjson_really_inline uint32_t string_to_uint32(const char* str) { uint32_t val
|
|||||||
|
|
||||||
// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive.
|
// Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive.
|
||||||
// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about.
|
// Yet all decent optimizing compilers will compile memcpy to a single instruction, just about.
|
||||||
|
// It is unsafe because you must ensure that you have at least 4 characters before calling this function!
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) {
|
simdjson_really_inline uint32_t str4ncmp_unsafe(const uint8_t *src, const char* atom) {
|
||||||
uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++)
|
uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++)
|
||||||
static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes");
|
|
||||||
std::memcpy(&srcval, src, sizeof(uint32_t));
|
std::memcpy(&srcval, src, sizeof(uint32_t));
|
||||||
return srcval ^ string_to_uint32(atom);
|
return srcval ^ string_to_uint32(atom);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// It is unsafe because you must ensure that you have at least 5 characters before calling this function!
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline bool is_valid_true_atom(const uint8_t *src) {
|
simdjson_really_inline bool is_valid_true_atom_unsafe(const uint8_t *src) {
|
||||||
return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
|
return (str4ncmp_unsafe(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) {
|
simdjson_really_inline bool is_valid_true_atom(const uint8_t *src, size_t len) {
|
||||||
if (len > 4) { return is_valid_true_atom(src); }
|
if (len > 4) { return is_valid_true_atom_unsafe(src); }
|
||||||
else if (len == 4) { return !str4ncmp(src, "true"); }
|
else if (len == 4) { return !str4ncmp_unsafe(src, "true"); }
|
||||||
else { return false; }
|
else { return false; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// It is unsafe because you must ensure that you have at least 5 characters before calling this function!
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline bool is_valid_false_atom(const uint8_t *src) {
|
simdjson_really_inline bool is_valid_false_atom_unsafe(const uint8_t *src) {
|
||||||
return (str4ncmp(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0;
|
return (str4ncmp_unsafe(src+1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) {
|
simdjson_really_inline bool is_valid_false_atom(const uint8_t *src, size_t len) {
|
||||||
if (len > 5) { return is_valid_false_atom(src); }
|
if (len > 5) { return is_valid_false_atom_unsafe(src); }
|
||||||
else if (len == 5) { return !str4ncmp(src+1, "alse"); }
|
else if (len == 5) { return !str4ncmp_unsafe(src+1, "alse"); }
|
||||||
else { return false; }
|
else { return false; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// It is unsafe because you must ensure that you have at least 5 characters before calling this function!
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline bool is_valid_null_atom(const uint8_t *src) {
|
simdjson_really_inline bool is_valid_null_atom_unsafe(const uint8_t *src) {
|
||||||
return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
|
return (str4ncmp_unsafe(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused
|
simdjson_warn_unused
|
||||||
simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) {
|
simdjson_really_inline bool is_valid_null_atom(const uint8_t *src, size_t len) {
|
||||||
if (len > 4) { return is_valid_null_atom(src); }
|
if (len > 4) { return is_valid_null_atom_unsafe(src); }
|
||||||
else if (len == 4) { return !str4ncmp(src, "null"); }
|
else if (len == 4) { return !str4ncmp_unsafe(src, "null"); }
|
||||||
else { return false; }
|
else { return false; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,27 +4,8 @@
|
|||||||
namespace simdjson {
|
namespace simdjson {
|
||||||
namespace SIMDJSON_IMPLEMENTATION {
|
namespace SIMDJSON_IMPLEMENTATION {
|
||||||
|
|
||||||
// expectation: sizeof(open_container) = 64/8.
|
|
||||||
struct open_container {
|
|
||||||
uint32_t tape_index; // where, on the tape, does the scope ([,{) begins
|
|
||||||
uint32_t count; // how many elements in the scope
|
|
||||||
}; // struct open_container
|
|
||||||
|
|
||||||
static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits");
|
|
||||||
|
|
||||||
class dom_parser_implementation final : public internal::dom_parser_implementation {
|
class dom_parser_implementation final : public internal::dom_parser_implementation {
|
||||||
public:
|
public:
|
||||||
/** Tape location of each open { or [ */
|
|
||||||
std::unique_ptr<open_container[]> open_containers{};
|
|
||||||
/** Whether each open container is a [ or { */
|
|
||||||
std::unique_ptr<bool[]> is_array{};
|
|
||||||
/** Buffer passed to stage 1 */
|
|
||||||
const uint8_t *buf{};
|
|
||||||
/** Length passed to stage 1 */
|
|
||||||
size_t len{0};
|
|
||||||
/** Document passed to stage 2 */
|
|
||||||
dom::document *doc{};
|
|
||||||
|
|
||||||
inline dom_parser_implementation() noexcept;
|
inline dom_parser_implementation() noexcept;
|
||||||
inline dom_parser_implementation(dom_parser_implementation &&other) noexcept;
|
inline dom_parser_implementation(dom_parser_implementation &&other) noexcept;
|
||||||
inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept;
|
inline dom_parser_implementation &operator=(dom_parser_implementation &&other) noexcept;
|
||||||
@@ -68,7 +49,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s
|
|||||||
|
|
||||||
inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept {
|
inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept {
|
||||||
// Stage 2 stacks
|
// Stage 2 stacks
|
||||||
open_containers.reset(new (std::nothrow) open_container[max_depth]);
|
open_containers.reset(new (std::nothrow) internal::open_container[max_depth]);
|
||||||
is_array.reset(new (std::nothrow) bool[max_depth]);
|
is_array.reset(new (std::nothrow) bool[max_depth]);
|
||||||
if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; }
|
if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; }
|
||||||
|
|
||||||
|
|||||||
@@ -291,8 +291,8 @@ simdjson_really_inline bool compute_float_64(int64_t power, uint64_t i, bool neg
|
|||||||
// It will return an error (false) if the parsed number is infinite.
|
// It will return an error (false) if the parsed number is infinite.
|
||||||
// The string parsing itself always succeeds. We know that there is at least
|
// The string parsing itself always succeeds. We know that there is at least
|
||||||
// one digit.
|
// one digit.
|
||||||
static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) {
|
bool parse_float_fallback(const uint8_t * const ptr, const uint8_t * const end, double *outDouble) {
|
||||||
*outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr));
|
*outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr), reinterpret_cast<const char *>(end));
|
||||||
// We do not accept infinite values.
|
// We do not accept infinite values.
|
||||||
|
|
||||||
// Detecting finite values in a portable manner is ridiculously hard, ideally
|
// Detecting finite values in a portable manner is ridiculously hard, ideally
|
||||||
@@ -305,29 +305,17 @@ static bool parse_float_fallback(const uint8_t *ptr, double *outDouble) {
|
|||||||
// to handle that max may be a macro on windows).
|
// to handle that max may be a macro on windows).
|
||||||
return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
|
return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
|
||||||
}
|
}
|
||||||
static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, double *outDouble) {
|
|
||||||
*outDouble = simdjson::internal::from_chars(reinterpret_cast<const char *>(ptr), reinterpret_cast<const char *>(end_ptr));
|
|
||||||
// We do not accept infinite values.
|
|
||||||
|
|
||||||
// Detecting finite values in a portable manner is ridiculously hard, ideally
|
|
||||||
// we would want to do:
|
|
||||||
// return !std::isfinite(*outDouble);
|
|
||||||
// but that mysteriously fails under legacy/old libc++ libraries, see
|
|
||||||
// https://github.com/simdjson/simdjson/issues/1286
|
|
||||||
//
|
|
||||||
// Therefore, fall back to this solution (the extra parens are there
|
|
||||||
// to handle that max may be a macro on windows).
|
|
||||||
return !(*outDouble > (std::numeric_limits<double>::max)() || *outDouble < std::numeric_limits<double>::lowest());
|
|
||||||
}
|
|
||||||
|
|
||||||
// check quickly whether the next 8 chars are made of digits
|
// check quickly whether the next 8 chars are made of digits
|
||||||
// at a glance, it looks better than Mula's
|
// at a glance, it looks better than Mula's
|
||||||
// http://0x80.pl/articles/swar-digits-validate.html
|
// http://0x80.pl/articles/swar-digits-validate.html
|
||||||
simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) {
|
simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t * const chars, const uint8_t * const end) {
|
||||||
uint64_t val;
|
uint64_t val;
|
||||||
// this can read up to 7 bytes beyond the buffer size, but we require
|
// end == nullptr is forbidden here since we have SIMDJSON_SWAR_NUMBER_PARSING
|
||||||
// SIMDJSON_PADDING of padding
|
#ifndef SIMDJSON_SWAR_NUMBER_PARSING
|
||||||
static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7");
|
#warning "You should never call is_made_of_eight_digits_fast given that SIMDJSON_SWAR_NUMBER_PARSING is undefined."
|
||||||
|
#endif
|
||||||
|
SIMDJSON_ASSUME(end != nullptr);
|
||||||
|
if ((end-chars) < 8) { return false; }
|
||||||
std::memcpy(&val, chars, 8);
|
std::memcpy(&val, chars, 8);
|
||||||
// a branchy method might be faster:
|
// a branchy method might be faster:
|
||||||
// return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030)
|
// return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030)
|
||||||
@@ -339,19 +327,34 @@ simdjson_really_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
template<typename W>
|
template<typename W>
|
||||||
error_code slow_float_parsing(simdjson_unused const uint8_t * src, W writer) {
|
error_code slow_float_parsing(const uint8_t * src, W writer) {
|
||||||
double d;
|
double d;
|
||||||
if (parse_float_fallback(src, &d)) {
|
if (parse_float_fallback(src, nullptr, &d)) {
|
||||||
writer.append_double(d);
|
writer.append_double(d);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
return INVALID_NUMBER(src);
|
return INVALID_NUMBER(src);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template<typename W>
|
||||||
|
error_code slow_float_parsing(const uint8_t * src, const uint8_t * const end, W writer) {
|
||||||
|
double d;
|
||||||
|
if (parse_float_fallback(src, end, &d)) {
|
||||||
|
writer.append_double(d);
|
||||||
|
return SUCCESS;
|
||||||
|
}
|
||||||
|
return INVALID_NUMBER(src);
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline bool at_end(const uint8_t * const p, const uint8_t * const end) {
|
||||||
|
return end != nullptr && p >= end;
|
||||||
|
}
|
||||||
|
|
||||||
template<typename I>
|
template<typename I>
|
||||||
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later
|
||||||
simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
simdjson_really_inline bool parse_digit(const uint8_t * const p, const uint8_t * const end, I &i) {
|
||||||
const uint8_t digit = static_cast<uint8_t>(c - '0');
|
if (at_end(p, end)) { return false; }
|
||||||
|
const uint8_t digit = static_cast<uint8_t>(*p - '0');
|
||||||
if (digit > 9) {
|
if (digit > 9) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@@ -360,24 +363,24 @@ simdjson_really_inline bool parse_digit(const uint8_t c, I &i) {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t * const src, const uint8_t * const end, const uint8_t *&p, uint64_t &i, int64_t &exponent) {
|
||||||
// we continue with the fiction that we have an integer. If the
|
// we continue with the fiction that we have an integer. If the
|
||||||
// floating point number is representable as x * 10^z for some integer
|
// floating point number is representable as x * 10^z for some integer
|
||||||
// z that fits in 53 bits, then we will be able to convert back the
|
// z that fits in 53 bits, then we will be able to convert back the
|
||||||
// the integer into a float in a lossless manner.
|
// the integer into a float in a lossless manner.
|
||||||
const uint8_t *const first_after_period = p;
|
const uint8_t * const first_after_period = p;
|
||||||
|
|
||||||
#ifdef SWAR_NUMBER_PARSING
|
#ifdef SIMDJSON_SWAR_NUMBER_PARSING
|
||||||
// this helps if we have lots of decimals!
|
// this helps if we have lots of decimals!
|
||||||
// this turns out to be frequent enough.
|
// this turns out to be frequent enough.
|
||||||
if (is_made_of_eight_digits_fast(p)) {
|
if (is_made_of_eight_digits_fast(p, end)) {
|
||||||
i = i * 100000000 + parse_eight_digits_unrolled(p);
|
i = i * 100000000 + parse_eight_digits_unrolled(p);
|
||||||
p += 8;
|
p += 8;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
// Unrolling the first digit makes a small difference on some implementations (e.g. westmere)
|
// Unrolling the first digit makes a small difference on some implementations (e.g. westmere)
|
||||||
if (parse_digit(*p, i)) { ++p; }
|
if (parse_digit(p, end, i)) { ++p; }
|
||||||
while (parse_digit(*p, i)) { p++; }
|
while (parse_digit(p, end, i)) { ++p; }
|
||||||
exponent = first_after_period - p;
|
exponent = first_after_period - p;
|
||||||
// Decimal without digits (123.) is illegal
|
// Decimal without digits (123.) is illegal
|
||||||
if (exponent == 0) {
|
if (exponent == 0) {
|
||||||
@@ -386,15 +389,15 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c
|
|||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *const src, const uint8_t *&p, int64_t &exponent) {
|
simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t * const src, const uint8_t * const end, const uint8_t *&p, int64_t &exponent) {
|
||||||
// Exp Sign: -123.456e[-]78
|
// Exp Sign: -123.456e[-]78
|
||||||
bool neg_exp = ('-' == *p);
|
bool neg_exp = (!at_end(p, end) && '-' == *p);
|
||||||
if (neg_exp || '+' == *p) { p++; } // Skip + as well
|
if (neg_exp || (!at_end(p, end) && '+' == *p)) { ++p; } // Skip + as well
|
||||||
|
|
||||||
// Exponent: -123.456e-[78]
|
// Exponent: -123.456e-[78]
|
||||||
auto start_exp = p;
|
auto start_exp = p;
|
||||||
int64_t exp_number = 0;
|
int64_t exp_number = 0;
|
||||||
while (parse_digit(*p, exp_number)) { ++p; }
|
while (parse_digit(p, end, exp_number)) { ++p; }
|
||||||
// It is possible for parse_digit to overflow.
|
// It is possible for parse_digit to overflow.
|
||||||
// In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN.
|
// In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN.
|
||||||
// Thus we *must* check for possible overflow before we negate exp_number.
|
// Thus we *must* check for possible overflow before we negate exp_number.
|
||||||
@@ -416,7 +419,7 @@ simdjson_really_inline error_code parse_exponent(simdjson_unused const uint8_t *
|
|||||||
// something!!!!
|
// something!!!!
|
||||||
if (simdjson_unlikely(p > start_exp+18)) {
|
if (simdjson_unlikely(p > start_exp+18)) {
|
||||||
// Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow
|
// Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow
|
||||||
while (*start_exp == '0') { start_exp++; }
|
while (*start_exp == '0') { ++start_exp; }
|
||||||
// 19 digits could overflow int64_t and is kind of absurd anyway. We don't
|
// 19 digits could overflow int64_t and is kind of absurd anyway. We don't
|
||||||
// support exponents smaller than -999,999,999,999,999,999 and bigger
|
// support exponents smaller than -999,999,999,999,999,999 and bigger
|
||||||
// than 999,999,999,999,999,999.
|
// than 999,999,999,999,999,999.
|
||||||
@@ -442,14 +445,14 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s
|
|||||||
// We have to handle the case where we have 0.0000somenumber.
|
// We have to handle the case where we have 0.0000somenumber.
|
||||||
const uint8_t *start = start_digits;
|
const uint8_t *start = start_digits;
|
||||||
while ((*start == '0') || (*start == '.')) {
|
while ((*start == '0') || (*start == '.')) {
|
||||||
start++;
|
++start;
|
||||||
}
|
}
|
||||||
// we over-decrement by one when there is a '.'
|
// we over-decrement by one when there is a '.'
|
||||||
return digit_count - size_t(start - start_digits);
|
return digit_count - size_t(start - start_digits);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename W>
|
template<typename W>
|
||||||
simdjson_really_inline error_code write_float(const uint8_t *const src, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) {
|
simdjson_really_inline error_code write_float(const uint8_t * const src, const uint8_t * const end, bool negative, uint64_t i, const uint8_t * start_digits, size_t digit_count, int64_t exponent, W &writer) {
|
||||||
// If we frequently had to deal with long strings of digits,
|
// If we frequently had to deal with long strings of digits,
|
||||||
// we could extend our code by using a 128-bit integer instead
|
// we could extend our code by using a 128-bit integer instead
|
||||||
// of a 64-bit integer. However, this is uncommon in practice.
|
// of a 64-bit integer. However, this is uncommon in practice.
|
||||||
@@ -470,7 +473,7 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
|
|||||||
// it, it would force it to be stored in memory, preventing the compiler from picking it apart
|
// it, it would force it to be stored in memory, preventing the compiler from picking it apart
|
||||||
// and putting into registers. i.e. if we pass it as reference, it gets slow.
|
// and putting into registers. i.e. if we pass it as reference, it gets slow.
|
||||||
// This is what forces the skip_double, as well.
|
// This is what forces the skip_double, as well.
|
||||||
error_code error = slow_float_parsing(src, writer);
|
error_code error = slow_float_parsing(src, end, writer);
|
||||||
writer.skip_double();
|
writer.skip_double();
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
@@ -495,7 +498,7 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
|
|||||||
double d;
|
double d;
|
||||||
if (!compute_float_64(exponent, i, negative, d)) {
|
if (!compute_float_64(exponent, i, negative, d)) {
|
||||||
// we are almost never going to get here.
|
// we are almost never going to get here.
|
||||||
if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); }
|
if (!parse_float_fallback(src, end, &d)) { return INVALID_NUMBER(src); }
|
||||||
}
|
}
|
||||||
WRITE_DOUBLE(d, src, writer);
|
WRITE_DOUBLE(d, src, writer);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
@@ -505,14 +508,14 @@ simdjson_really_inline error_code write_float(const uint8_t *const src, bool neg
|
|||||||
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
#ifdef SIMDJSON_SKIPNUMBERPARSING
|
||||||
|
|
||||||
template<typename W>
|
template<typename W>
|
||||||
simdjson_really_inline error_code parse_number(const uint8_t *const, W &writer) {
|
simdjson_really_inline error_code parse_number(const uint8_t * const, W &writer, const uint8_t * const) {
|
||||||
writer.append_s64(0); // always write zero
|
writer.append_s64(0); // always write zero
|
||||||
return SUCCESS; // always succeeds
|
return SUCCESS; // always succeeds
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src) noexcept { return 0; }
|
simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept { return 0; }
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src) noexcept { return 0; }
|
simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept { return 0; }
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * const src) noexcept { return 0; }
|
simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept { return 0; }
|
||||||
|
|
||||||
#else
|
#else
|
||||||
|
|
||||||
@@ -526,8 +529,8 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
|
|||||||
//
|
//
|
||||||
// Our objective is accurate parsing (ULP of 0) at high speed.
|
// Our objective is accurate parsing (ULP of 0) at high speed.
|
||||||
template<typename W>
|
template<typename W>
|
||||||
simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writer) {
|
simdjson_really_inline error_code parse_number(const uint8_t * const src, W &writer, const uint8_t * const end) {
|
||||||
|
SIMDJSON_ASSUME(end != nullptr);
|
||||||
//
|
//
|
||||||
// Check for minus sign
|
// Check for minus sign
|
||||||
//
|
//
|
||||||
@@ -538,9 +541,9 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
|
|||||||
// Parse the integer part.
|
// Parse the integer part.
|
||||||
//
|
//
|
||||||
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
||||||
const uint8_t *const start_digits = p;
|
const uint8_t * const start_digits = p;
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
while (parse_digit(*p, i)) { p++; }
|
while (parse_digit(p, end, i)) { ++p; }
|
||||||
|
|
||||||
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
||||||
// Optimization note: size_t is expected to be unsigned.
|
// Optimization note: size_t is expected to be unsigned.
|
||||||
@@ -552,20 +555,20 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
|
|||||||
//
|
//
|
||||||
int64_t exponent = 0;
|
int64_t exponent = 0;
|
||||||
bool is_float = false;
|
bool is_float = false;
|
||||||
if ('.' == *p) {
|
if (!at_end(p, end) && '.' == *p) {
|
||||||
is_float = true;
|
is_float = true;
|
||||||
++p;
|
++p;
|
||||||
SIMDJSON_TRY( parse_decimal(src, p, i, exponent) );
|
SIMDJSON_TRY( parse_decimal(src, end, p, i, exponent) );
|
||||||
digit_count = int(p - start_digits); // used later to guard against overflows
|
digit_count = int(p - start_digits); // used later to guard against overflows
|
||||||
}
|
}
|
||||||
if (('e' == *p) || ('E' == *p)) {
|
if (!at_end(p, end) && ('e' == *p || 'E' == *p)) {
|
||||||
is_float = true;
|
is_float = true;
|
||||||
++p;
|
++p;
|
||||||
SIMDJSON_TRY( parse_exponent(src, p, exponent) );
|
SIMDJSON_TRY( parse_exponent(src, end, p, exponent) );
|
||||||
}
|
}
|
||||||
if (is_float) {
|
if (is_float) {
|
||||||
const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p);
|
const bool dirty_end = !at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p);
|
||||||
SIMDJSON_TRY( write_float(src, negative, i, start_digits, digit_count, exponent, writer) );
|
SIMDJSON_TRY( write_float(src, end, negative, i, start_digits, digit_count, exponent, writer) );
|
||||||
if (dirty_end) { return INVALID_NUMBER(src); }
|
if (dirty_end) { return INVALID_NUMBER(src); }
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
@@ -580,7 +583,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
|
|||||||
// Anything negative above INT64_MAX+1 is invalid
|
// Anything negative above INT64_MAX+1 is invalid
|
||||||
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
if (i > uint64_t(INT64_MAX)+1) { return INVALID_NUMBER(src); }
|
||||||
WRITE_INTEGER(~i+1, src, writer);
|
WRITE_INTEGER(~i+1, src, writer);
|
||||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
if (!at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
// Positive overflow check:
|
// Positive overflow check:
|
||||||
// - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
|
// - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
|
||||||
@@ -603,7 +606,7 @@ simdjson_really_inline error_code parse_number(const uint8_t *const src, W &writ
|
|||||||
} else {
|
} else {
|
||||||
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
|
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
|
||||||
}
|
}
|
||||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
if (!at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -676,15 +679,15 @@ const uint8_t integer_string_finisher[256] = {
|
|||||||
NUMBER_ERROR};
|
NUMBER_ERROR};
|
||||||
|
|
||||||
// Parse any number from 0 to 18,446,744,073,709,551,615
|
// Parse any number from 0 to 18,446,744,073,709,551,615
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src) noexcept {
|
simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src, const uint8_t * const end = nullptr) noexcept {
|
||||||
const uint8_t *p = src;
|
const uint8_t *p = src;
|
||||||
//
|
//
|
||||||
// Parse the integer part.
|
// Parse the integer part.
|
||||||
//
|
//
|
||||||
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
||||||
const uint8_t *const start_digits = p;
|
const uint8_t * const start_digits = p;
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
while (parse_digit(*p, i)) { p++; }
|
while (parse_digit(p, end, i)) { ++p; }
|
||||||
|
|
||||||
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
||||||
// Optimization note: size_t is expected to be unsigned.
|
// Optimization note: size_t is expected to be unsigned.
|
||||||
@@ -702,7 +705,7 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
|
|||||||
// return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
|
// return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
|
||||||
// }
|
// }
|
||||||
// as a single table lookup:
|
// as a single table lookup:
|
||||||
if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
|
if (!at_end(p, end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
|
||||||
|
|
||||||
if (digit_count == 20) {
|
if (digit_count == 20) {
|
||||||
// Positive overflow check:
|
// Positive overflow check:
|
||||||
@@ -724,70 +727,21 @@ simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// Parse any number from 0 to 18,446,744,073,709,551,615
|
|
||||||
// Never read at src_end or beyond
|
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<uint64_t> parse_unsigned(const uint8_t * const src, const uint8_t * const src_end) noexcept {
|
|
||||||
const uint8_t *p = src;
|
|
||||||
//
|
|
||||||
// Parse the integer part.
|
|
||||||
//
|
|
||||||
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
|
||||||
const uint8_t *const start_digits = p;
|
|
||||||
uint64_t i = 0;
|
|
||||||
while ((p != src_end) && parse_digit(*p, i)) { p++; }
|
|
||||||
|
|
||||||
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
|
||||||
// Optimization note: size_t is expected to be unsigned.
|
|
||||||
size_t digit_count = size_t(p - start_digits);
|
|
||||||
// The longest positive 64-bit number is 20 digits.
|
|
||||||
// We do it this way so we don't trigger this branch unless we must.
|
|
||||||
// Optimization note: the compiler can probably merge
|
|
||||||
// ((digit_count == 0) || (digit_count > 20))
|
|
||||||
// into a single branch since digit_count is unsigned.
|
|
||||||
if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; }
|
|
||||||
// Here digit_count > 0.
|
|
||||||
if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
|
|
||||||
// We can do the following...
|
|
||||||
// if (!jsoncharutils::is_structural_or_whitespace(*p)) {
|
|
||||||
// return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
|
|
||||||
// }
|
|
||||||
// as a single table lookup:
|
|
||||||
if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
|
|
||||||
|
|
||||||
if (digit_count == 20) {
|
|
||||||
// Positive overflow check:
|
|
||||||
// - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the
|
|
||||||
// biggest uint64_t.
|
|
||||||
// - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX.
|
|
||||||
// If we got here, it's a 20 digit number starting with the digit "1".
|
|
||||||
// - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller
|
|
||||||
// than 1,553,255,926,290,448,384.
|
|
||||||
// - That is smaller than the smallest possible 20-digit number the user could write:
|
|
||||||
// 10,000,000,000,000,000,000.
|
|
||||||
// - Therefore, if the number is positive and lower than that, it's overflow.
|
|
||||||
// - The value we are looking at is less than or equal to 9,223,372,036,854,775,808 (INT64_MAX).
|
|
||||||
//
|
|
||||||
if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; }
|
|
||||||
}
|
|
||||||
|
|
||||||
return i;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
|
// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t *src) noexcept {
|
simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * src, const uint8_t * const end = nullptr) noexcept {
|
||||||
//
|
//
|
||||||
// Check for minus sign
|
// Check for minus sign
|
||||||
//
|
//
|
||||||
bool negative = (*src == '-');
|
bool negative = (!at_end(src, end) && *src == '-');
|
||||||
const uint8_t *p = src + negative;
|
const uint8_t *p = src + negative;
|
||||||
|
|
||||||
//
|
//
|
||||||
// Parse the integer part.
|
// Parse the integer part.
|
||||||
//
|
//
|
||||||
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
||||||
const uint8_t *const start_digits = p;
|
const uint8_t * const start_digits = p;
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
while (parse_digit(*p, i)) { p++; }
|
while (parse_digit(p, end, i)) { ++p; }
|
||||||
|
|
||||||
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
||||||
// Optimization note: size_t is expected to be unsigned.
|
// Optimization note: size_t is expected to be unsigned.
|
||||||
@@ -807,7 +761,7 @@ simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(co
|
|||||||
// return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
|
// return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
|
||||||
// }
|
// }
|
||||||
// as a single table lookup:
|
// as a single table lookup:
|
||||||
if(integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
|
if (!at_end(p, end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
|
||||||
// Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
|
// Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
|
||||||
// Performance note: This check is only needed when digit_count == longest_digit_count but it is
|
// Performance note: This check is only needed when digit_count == longest_digit_count but it is
|
||||||
// so cheap that we might as well always make it.
|
// so cheap that we might as well always make it.
|
||||||
@@ -815,55 +769,11 @@ simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(co
|
|||||||
return negative ? (~i+1) : i;
|
return negative ? (~i+1) : i;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
|
simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src, const uint8_t * const end = nullptr) noexcept {
|
||||||
// Never read at src_end or beyond
|
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<int64_t> parse_integer(const uint8_t * const src, const uint8_t * const src_end) noexcept {
|
|
||||||
//
|
//
|
||||||
// Check for minus sign
|
// Check for minus sign
|
||||||
//
|
//
|
||||||
if(src == src_end) { return NUMBER_ERROR; }
|
bool negative = (!at_end(src, end) && *src == '-');
|
||||||
bool negative = (*src == '-');
|
|
||||||
const uint8_t *p = src + negative;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Parse the integer part.
|
|
||||||
//
|
|
||||||
// PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare
|
|
||||||
const uint8_t *const start_digits = p;
|
|
||||||
uint64_t i = 0;
|
|
||||||
while ((p != src_end) && parse_digit(*p, i)) { p++; }
|
|
||||||
|
|
||||||
// If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error.
|
|
||||||
// Optimization note: size_t is expected to be unsigned.
|
|
||||||
size_t digit_count = size_t(p - start_digits);
|
|
||||||
// We go from
|
|
||||||
// -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807
|
|
||||||
// so we can never represent numbers that have more than 19 digits.
|
|
||||||
size_t longest_digit_count = 19;
|
|
||||||
// Optimization note: the compiler can probably merge
|
|
||||||
// ((digit_count == 0) || (digit_count > longest_digit_count))
|
|
||||||
// into a single branch since digit_count is unsigned.
|
|
||||||
if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; }
|
|
||||||
// Here digit_count > 0.
|
|
||||||
if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; }
|
|
||||||
// We can do the following...
|
|
||||||
// if (!jsoncharutils::is_structural_or_whitespace(*p)) {
|
|
||||||
// return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR;
|
|
||||||
// }
|
|
||||||
// as a single table lookup:
|
|
||||||
if((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); }
|
|
||||||
// Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX.
|
|
||||||
// Performance note: This check is only needed when digit_count == longest_digit_count but it is
|
|
||||||
// so cheap that we might as well always make it.
|
|
||||||
if(i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; }
|
|
||||||
return negative ? (~i+1) : i;
|
|
||||||
}
|
|
||||||
|
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src) noexcept {
|
|
||||||
//
|
|
||||||
// Check for minus sign
|
|
||||||
//
|
|
||||||
bool negative = (*src == '-');
|
|
||||||
src += negative;
|
src += negative;
|
||||||
|
|
||||||
//
|
//
|
||||||
@@ -871,9 +781,9 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
|
|||||||
//
|
//
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
const uint8_t *p = src;
|
const uint8_t *p = src;
|
||||||
p += parse_digit(*p, i);
|
p += parse_digit(p, end, i);
|
||||||
bool leading_zero = (i == 0);
|
bool leading_zero = (i == 0);
|
||||||
while (parse_digit(*p, i)) { p++; }
|
while (parse_digit(p, end, i)) { ++p; }
|
||||||
// no integer digits, or 0123 (zero must be solo)
|
// no integer digits, or 0123 (zero must be solo)
|
||||||
if ( p == src ) { return INCORRECT_TYPE; }
|
if ( p == src ) { return INCORRECT_TYPE; }
|
||||||
if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
|
if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
|
||||||
@@ -881,14 +791,20 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
|
|||||||
//
|
//
|
||||||
// Parse the decimal part.
|
// Parse the decimal part.
|
||||||
//
|
//
|
||||||
|
// performance todo: we could use SWAR here to quickly processing
|
||||||
|
// digits in blocs of, say, eight digits. Doing so requires that
|
||||||
|
// we always have access to some estimation of the end of the buffer.
|
||||||
|
// Currently, we only have that for root numbers in the On Demand API,
|
||||||
|
// so it is not terribly useful for performance purposes.
|
||||||
|
//
|
||||||
int64_t exponent = 0;
|
int64_t exponent = 0;
|
||||||
bool overflow;
|
bool overflow;
|
||||||
if (simdjson_likely(*p == '.')) {
|
if (simdjson_likely(!at_end(p, end) && '.' == *p)) {
|
||||||
p++;
|
++p;
|
||||||
const uint8_t *start_decimal_digits = p;
|
const uint8_t *start_decimal_digits = p;
|
||||||
if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits
|
if (!parse_digit(p, end, i)) { return NUMBER_ERROR; } // no decimal digits
|
||||||
p++;
|
++p;
|
||||||
while (parse_digit(*p, i)) { p++; }
|
while (parse_digit(p, end, i)) { ++p; }
|
||||||
exponent = -(p - start_decimal_digits);
|
exponent = -(p - start_decimal_digits);
|
||||||
|
|
||||||
// Overflow check. More than 19 digits (minus the decimal) may be overflow.
|
// Overflow check. More than 19 digits (minus the decimal) may be overflow.
|
||||||
@@ -896,7 +812,7 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
|
|||||||
if (simdjson_unlikely(overflow && leading_zero)) {
|
if (simdjson_unlikely(overflow && leading_zero)) {
|
||||||
// Skip leading 0.00000 and see if it still overflows
|
// Skip leading 0.00000 and see if it still overflows
|
||||||
const uint8_t *start_digits = src + 2;
|
const uint8_t *start_digits = src + 2;
|
||||||
while (*start_digits == '0') { start_digits++; }
|
while (!at_end(start_digits, end) && *start_digits == '0') { ++start_digits; }
|
||||||
overflow = start_digits-src > 19;
|
overflow = start_digits-src > 19;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -906,21 +822,21 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
|
|||||||
//
|
//
|
||||||
// Parse the exponent
|
// Parse the exponent
|
||||||
//
|
//
|
||||||
if (*p == 'e' || *p == 'E') {
|
if (!at_end(p, end) && ('e' == *p || 'E' == *p)) {
|
||||||
p++;
|
++p;
|
||||||
bool exp_neg = *p == '-';
|
bool exp_neg = (!at_end(p, end) && '-' == *p);
|
||||||
p += exp_neg || *p == '+';
|
p += exp_neg || (!at_end(p, end) && '+' == *p);
|
||||||
|
|
||||||
uint64_t exp = 0;
|
uint64_t exp = 0;
|
||||||
const uint8_t *start_exp_digits = p;
|
const uint8_t *start_exp_digits = p;
|
||||||
while (parse_digit(*p, exp)) { p++; }
|
while (parse_digit(p, end, exp)) { ++p; }
|
||||||
// no exp digits, or 20+ exp digits
|
// no exp digits, or 20+ exp digits
|
||||||
if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
|
if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
|
||||||
|
|
||||||
exponent += exp_neg ? 0-exp : exp;
|
exponent += exp_neg ? 0-exp : exp;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
|
if (!at_end(p, end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
|
||||||
|
|
||||||
overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
|
overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
|
||||||
|
|
||||||
@@ -931,90 +847,7 @@ simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(cons
|
|||||||
if (simdjson_likely(!overflow)) {
|
if (simdjson_likely(!overflow)) {
|
||||||
if (compute_float_64(exponent, i, negative, d)) { return d; }
|
if (compute_float_64(exponent, i, negative, d)) { return d; }
|
||||||
}
|
}
|
||||||
if (!parse_float_fallback(src-negative, &d)) {
|
if (!parse_float_fallback(src-negative, end, &d)) {
|
||||||
return NUMBER_ERROR;
|
|
||||||
}
|
|
||||||
return d;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
// Never read at src_end or beyond
|
|
||||||
simdjson_unused simdjson_really_inline simdjson_result<double> parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept {
|
|
||||||
if(src == src_end) { return NUMBER_ERROR; }
|
|
||||||
//
|
|
||||||
// Check for minus sign
|
|
||||||
//
|
|
||||||
bool negative = (*src == '-');
|
|
||||||
src += negative;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Parse the integer part.
|
|
||||||
//
|
|
||||||
uint64_t i = 0;
|
|
||||||
const uint8_t *p = src;
|
|
||||||
if(p == src_end) { return NUMBER_ERROR; }
|
|
||||||
p += parse_digit(*p, i);
|
|
||||||
bool leading_zero = (i == 0);
|
|
||||||
while ((p != src_end) && parse_digit(*p, i)) { p++; }
|
|
||||||
// no integer digits, or 0123 (zero must be solo)
|
|
||||||
if ( p == src ) { return INCORRECT_TYPE; }
|
|
||||||
if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; }
|
|
||||||
|
|
||||||
//
|
|
||||||
// Parse the decimal part.
|
|
||||||
//
|
|
||||||
int64_t exponent = 0;
|
|
||||||
bool overflow;
|
|
||||||
if (simdjson_likely((p != src_end) && (*p == '.'))) {
|
|
||||||
p++;
|
|
||||||
const uint8_t *start_decimal_digits = p;
|
|
||||||
if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits
|
|
||||||
p++;
|
|
||||||
while ((p != src_end) && parse_digit(*p, i)) { p++; }
|
|
||||||
exponent = -(p - start_decimal_digits);
|
|
||||||
|
|
||||||
// Overflow check. More than 19 digits (minus the decimal) may be overflow.
|
|
||||||
overflow = p-src-1 > 19;
|
|
||||||
if (simdjson_unlikely(overflow && leading_zero)) {
|
|
||||||
// Skip leading 0.00000 and see if it still overflows
|
|
||||||
const uint8_t *start_digits = src + 2;
|
|
||||||
while (*start_digits == '0') { start_digits++; }
|
|
||||||
overflow = start_digits-src > 19;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
overflow = p-src > 19;
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
|
||||||
// Parse the exponent
|
|
||||||
//
|
|
||||||
if ((p != src_end) && (*p == 'e' || *p == 'E')) {
|
|
||||||
p++;
|
|
||||||
if(p == src_end) { return NUMBER_ERROR; }
|
|
||||||
bool exp_neg = *p == '-';
|
|
||||||
p += exp_neg || *p == '+';
|
|
||||||
|
|
||||||
uint64_t exp = 0;
|
|
||||||
const uint8_t *start_exp_digits = p;
|
|
||||||
while ((p != src_end) && parse_digit(*p, exp)) { p++; }
|
|
||||||
// no exp digits, or 20+ exp digits
|
|
||||||
if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; }
|
|
||||||
|
|
||||||
exponent += exp_neg ? 0-exp : exp;
|
|
||||||
}
|
|
||||||
|
|
||||||
if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; }
|
|
||||||
|
|
||||||
overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power;
|
|
||||||
|
|
||||||
//
|
|
||||||
// Assemble (or slow-parse) the float
|
|
||||||
//
|
|
||||||
double d;
|
|
||||||
if (simdjson_likely(!overflow)) {
|
|
||||||
if (compute_float_64(exponent, i, negative, d)) { return d; }
|
|
||||||
}
|
|
||||||
if (!parse_float_fallback(src-negative, src_end, &d)) {
|
|
||||||
return NUMBER_ERROR;
|
return NUMBER_ERROR;
|
||||||
}
|
}
|
||||||
return d;
|
return d;
|
||||||
|
|||||||
@@ -82,7 +82,10 @@ simdjson_really_inline simdjson_result<std::string_view> array::raw_json() noexc
|
|||||||
const uint8_t * starting_point{iter.peek_start()};
|
const uint8_t * starting_point{iter.peek_start()};
|
||||||
auto error = consume();
|
auto error = consume();
|
||||||
if(error) { return error; }
|
if(error) { return error; }
|
||||||
const uint8_t * final_point{iter._json_iter->peek(0)};
|
// After 'consume()', we could be left pointing just beyond the document, but that
|
||||||
|
// is ok because we are not going to dereference the final pointer position, we just
|
||||||
|
// use it to compute the length in bytes.
|
||||||
|
const uint8_t * final_point{iter._json_iter->unsafe_pointer()};
|
||||||
return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
|
return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -156,12 +156,14 @@ simdjson_really_inline error_code document::consume() noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline simdjson_result<std::string_view> document::raw_json() noexcept {
|
simdjson_really_inline simdjson_result<std::string_view> document::raw_json() noexcept {
|
||||||
printf("document::raw_json()\n");
|
|
||||||
auto _iter = get_root_value_iterator();
|
auto _iter = get_root_value_iterator();
|
||||||
const uint8_t * starting_point{_iter.peek_start()};
|
const uint8_t * starting_point{_iter.peek_start()};
|
||||||
auto error = consume();
|
auto error = consume();
|
||||||
if(error) { return error; }
|
if(error) { return error; }
|
||||||
const uint8_t * final_point{iter.peek(0)};
|
// After 'consume()', we could be left pointing just beyond the document, but that
|
||||||
|
// is ok because we are not going to dereference the final pointer position, we just
|
||||||
|
// use it to compute the length in bytes.
|
||||||
|
const uint8_t * final_point{iter.unsafe_pointer()};
|
||||||
return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
|
return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -345,7 +345,7 @@ simdjson_really_inline std::string_view document_stream::iterator::source() cons
|
|||||||
cur_struct_index++;
|
cur_struct_index++;
|
||||||
}
|
}
|
||||||
|
|
||||||
return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[cur_struct_index] - current_index() + stream->batch_start + 1);;
|
return std::string_view(reinterpret_cast<const char*>(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[cur_struct_index] - current_index() + stream->batch_start + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline error_code document_stream::iterator::error() const noexcept {
|
inline error_code document_stream::iterator::error() const noexcept {
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ simdjson_really_inline simdjson_result<field> field::start(const value_iterator
|
|||||||
|
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> field::unescaped_key() noexcept {
|
||||||
SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us.
|
SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us.
|
||||||
simdjson_result<std::string_view> answer = first.unescape(second.iter.string_buf_loc());
|
simdjson_result<std::string_view> answer = first.unescape(*second.iter._json_iter);
|
||||||
first.consume();
|
first.consume();
|
||||||
return answer;
|
return answer;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ simdjson_really_inline json_iterator::json_iterator(const uint8_t *buf, ondemand
|
|||||||
|
|
||||||
inline void json_iterator::rewind() noexcept {
|
inline void json_iterator::rewind() noexcept {
|
||||||
token.set_position( root_position() );
|
token.set_position( root_position() );
|
||||||
|
assert_more_tokens(1);
|
||||||
logger::log_headers(); // We start again
|
logger::log_headers(); // We start again
|
||||||
_string_buf_loc = parser->string_buf.get();
|
_string_buf_loc = parser->string_buf.get();
|
||||||
_depth = 1;
|
_depth = 1;
|
||||||
@@ -63,6 +64,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
|
|||||||
* check.
|
* check.
|
||||||
*/
|
*/
|
||||||
if (depth() <= parent_depth) { return SUCCESS; }
|
if (depth() <= parent_depth) { return SUCCESS; }
|
||||||
|
SIMDJSON_TRY( require_tokens(1) );
|
||||||
switch (*return_current_and_advance()) {
|
switch (*return_current_and_advance()) {
|
||||||
// TODO consider whether matching braces is a requirement: if non-matching braces indicates
|
// TODO consider whether matching braces is a requirement: if non-matching braces indicates
|
||||||
// *missing* braces, then future lookups are not in the object/arrays they think they are,
|
// *missing* braces, then future lookups are not in the object/arrays they think they are,
|
||||||
@@ -84,10 +86,10 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
|
|||||||
logger::log_end_value(*this, "skip");
|
logger::log_end_value(*this, "skip");
|
||||||
_depth--;
|
_depth--;
|
||||||
if (depth() <= parent_depth) { return SUCCESS; }
|
if (depth() <= parent_depth) { return SUCCESS; }
|
||||||
#if __SIMDJSON_CHECK_EOF
|
#if SIMDJSON_CHECK_EOF
|
||||||
// If there are no more tokens, the parent is incomplete.
|
// If there are no more tokens, the parent is incomplete.
|
||||||
if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
|
if (at_end_of_input_buffer()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); }
|
||||||
#endif // __SIMDJSON_CHECK_EOF
|
#endif // SIMDJSON_CHECK_EOF
|
||||||
break;
|
break;
|
||||||
/*case '"':
|
/*case '"':
|
||||||
if(*peek() == ':') {
|
if(*peek() == ':') {
|
||||||
@@ -112,7 +114,7 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Now that we've considered the first value, we only increment/decrement for arrays/objects
|
// Now that we've considered the first value, we only increment/decrement for arrays/objects
|
||||||
while (position() < end_position()) {
|
while (position() < end_of_input_buffer_position()) {
|
||||||
switch (*return_current_and_advance()) {
|
switch (*return_current_and_advance()) {
|
||||||
case '[': case '{':
|
case '[': case '{':
|
||||||
logger::log_start_value(*this, "skip");
|
logger::log_start_value(*this, "skip");
|
||||||
@@ -161,7 +163,15 @@ simdjson_really_inline void json_iterator::assert_at_root() const noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept {
|
simdjson_really_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept {
|
||||||
assert_valid_position(token._position + required_tokens - 1);
|
#if SIMDJSON_CHECK_EOF
|
||||||
|
assert_valid_position(token.position() + required_tokens - 1);
|
||||||
|
#else
|
||||||
|
// We only check that we are at a valid position when SIMDJSON_CHECK_EOF is true,
|
||||||
|
// but otherwise, we will still end up with the 'required_tokens' parameter and
|
||||||
|
// compilers are bound to complain that it might be 'unused' (rightly so). Hence
|
||||||
|
// the following line is used to silence compiler warnings:
|
||||||
|
(void)required_tokens;
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline void json_iterator::assert_valid_position(token_position position) const noexcept {
|
simdjson_really_inline void json_iterator::assert_valid_position(token_position position) const noexcept {
|
||||||
@@ -171,22 +181,26 @@ simdjson_really_inline void json_iterator::assert_valid_position(token_position
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool json_iterator::at_end() const noexcept {
|
simdjson_really_inline bool json_iterator::at_end_of_input_buffer() const noexcept {
|
||||||
return position() == end_position();
|
return position() == end_of_input_buffer_position();
|
||||||
}
|
}
|
||||||
simdjson_really_inline token_position json_iterator::end_position() const noexcept {
|
simdjson_really_inline token_position json_iterator::end_of_input_buffer_position() const noexcept {
|
||||||
uint32_t n_structural_indexes{parser->implementation->n_structural_indexes};
|
uint32_t n_structural_indexes{parser->implementation->n_structural_indexes};
|
||||||
return &parser->implementation->structural_indexes[n_structural_indexes];
|
return &parser->implementation->structural_indexes[n_structural_indexes];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline const uint8_t *json_iterator::end_of_input_buffer() const noexcept {
|
||||||
|
return token.buf + parser->implementation->len;
|
||||||
|
}
|
||||||
|
|
||||||
inline std::string json_iterator::to_string() const noexcept {
|
inline std::string json_iterator::to_string() const noexcept {
|
||||||
if( !is_alive() ) { return "dead json_iterator instance"; }
|
if( !is_alive() ) { return "\"dead json_iterator instance\""; }
|
||||||
const char * current_structural = reinterpret_cast<const char *>(token.peek());
|
const char * current_structural = reinterpret_cast<const char *>(token.peek());
|
||||||
return std::string("json_iterator [ depth : ") + std::to_string(_depth)
|
return std::string("{\"name\": \"json_iterator\", \"depth\" : ") + std::to_string(_depth)
|
||||||
+ std::string(", structural : '") + std::string(current_structural,1)
|
+ std::string(", \"structural\" : \"") + std::string(current_structural,1)
|
||||||
+ std::string("', offset : ") + std::to_string(token.current_offset())
|
+ std::string("\", \"offset\" : ") + std::to_string(token.current_offset())
|
||||||
+ std::string("', error : ") + error_message(error)
|
+ std::string(", \"error\" : \"") + error_message(error)
|
||||||
+ std::string(" ]");
|
+ std::string("\"}");
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool json_iterator::is_alive() const noexcept {
|
simdjson_really_inline bool json_iterator::is_alive() const noexcept {
|
||||||
@@ -199,18 +213,33 @@ simdjson_really_inline void json_iterator::abandon() noexcept {
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept {
|
simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept {
|
||||||
// The following assert_more_tokens is currently disabled because rely on end-of-file buffering.
|
assert_more_tokens();
|
||||||
// assert_more_tokens();
|
|
||||||
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
|
|
||||||
// is ON by default, we have no choice but to disable it for real with a comment.
|
|
||||||
return token.return_current_and_advance();
|
return token.return_current_and_advance();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline simdjson_result<const uint8_t *> json_iterator::try_return_current_and_advance(uint32_t required_tokens) noexcept {
|
||||||
|
const uint8_t *json = token.return_current_and_advance();
|
||||||
|
// Check this *after* we get the pointer, since getting the pointer is more time-sensitive than the branch.
|
||||||
|
// Also resolves nicely to 0 in the common case of required_tokens == 1.
|
||||||
|
SIMDJSON_TRY( require_tokens(required_tokens - 1) );
|
||||||
|
return json;
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline error_code json_iterator::require_tokens(simdjson_unused uint32_t required_tokens) noexcept {
|
||||||
|
#if SIMDJSON_CHECK_EOF
|
||||||
|
if (position() + required_tokens > end_of_input_buffer_position()) {
|
||||||
|
return report_error(TAPE_ERROR, "Document ended early");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
return SUCCESS;
|
||||||
|
}
|
||||||
|
simdjson_really_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept {
|
||||||
|
// deliberately done without safety guard:
|
||||||
|
return token.peek(0);
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {
|
simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept {
|
||||||
// The following assert_more_tokens is currently disabled because rely on end-of-file buffering.
|
assert_more_tokens(delta+1);
|
||||||
// assert_more_tokens(delta+1);
|
|
||||||
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
|
|
||||||
// is ON by default, we have no choice but to disable it for real with a comment.
|
|
||||||
return token.peek(delta);
|
return token.peek(delta);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -220,20 +249,12 @@ simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline const uint8_t *json_iterator::peek(token_position position) const noexcept {
|
simdjson_really_inline const uint8_t *json_iterator::peek(token_position position) const noexcept {
|
||||||
// todo: currently we require end-of-string buffering, but the following
|
assert_valid_position(position);
|
||||||
// assert_valid_position should be turned on if/when we lift that condition.
|
|
||||||
// assert_valid_position(position);
|
|
||||||
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
|
|
||||||
// is ON by default, we have no choice but to disable it for real with a comment.
|
|
||||||
return token.peek(position);
|
return token.peek(position);
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept {
|
simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept {
|
||||||
// todo: currently we require end-of-string buffering, but the following
|
assert_valid_position(position);
|
||||||
// assert_valid_position should be turned on if/when we lift that condition.
|
|
||||||
// assert_valid_position(position);
|
|
||||||
// This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF
|
|
||||||
// is ON by default, we have no choice but to disable it for real with a comment.
|
|
||||||
return token.peek_length(position);
|
return token.peek_length(position);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -311,22 +332,6 @@ simdjson_really_inline error_code json_iterator::optional_error(error_code _erro
|
|||||||
return _error;
|
return _error;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<int N>
|
|
||||||
simdjson_warn_unused simdjson_really_inline bool json_iterator::copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept {
|
|
||||||
// Let us guard against silly cases:
|
|
||||||
if((N < max_len) || (N == 0)) { return false; }
|
|
||||||
// Truncate whitespace to fit the buffer.
|
|
||||||
if (max_len > N-1) {
|
|
||||||
// if (jsoncharutils::is_not_structural_or_whitespace(json[N-1])) { return false; }
|
|
||||||
max_len = N-1;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Copy to the buffer.
|
|
||||||
std::memcpy(tmpbuf, json, max_len);
|
|
||||||
tmpbuf[max_len] = ' ';
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace ondemand
|
} // namespace ondemand
|
||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
} // namespace simdjson
|
} // namespace simdjson
|
||||||
|
|||||||
@@ -95,9 +95,9 @@ public:
|
|||||||
simdjson_really_inline void assert_at_root() const noexcept;
|
simdjson_really_inline void assert_at_root() const noexcept;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tell whether the iterator is at the EOF mark
|
* Tell whether the iterator is at the EOF mark (end of the input buffer)
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline bool at_end() const noexcept;
|
simdjson_really_inline bool at_end_of_input_buffer() const noexcept;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tell whether the iterator is live (has not been moved).
|
* Tell whether the iterator is live (has not been moved).
|
||||||
@@ -114,6 +114,22 @@ public:
|
|||||||
*/
|
*/
|
||||||
simdjson_really_inline const uint8_t *return_current_and_advance() noexcept;
|
simdjson_really_inline const uint8_t *return_current_and_advance() noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Advance the current token by one, without modifying depth.
|
||||||
|
*
|
||||||
|
* @param requiremed_tokens The number of tokens that must exist. Only advances one token.
|
||||||
|
*
|
||||||
|
* @error TAPE_ERROR If there are not at least required_tokens tokens remaining.
|
||||||
|
*/
|
||||||
|
simdjson_really_inline simdjson_result<const uint8_t *> try_return_current_and_advance(uint32_t required_tokens=1) noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return an error unless there are enough tokens left.
|
||||||
|
*
|
||||||
|
* @param required_tokens The number of tokens that must exist.
|
||||||
|
* @error TAPE_ERROR If there are not at least required_tokens tokens remaining.
|
||||||
|
*/
|
||||||
|
simdjson_really_inline error_code require_tokens(uint32_t required_tokens=1) noexcept;
|
||||||
/**
|
/**
|
||||||
* Assert that there are at least the given number of tokens left.
|
* Assert that there are at least the given number of tokens left.
|
||||||
*
|
*
|
||||||
@@ -150,18 +166,24 @@ public:
|
|||||||
*
|
*
|
||||||
* This is not null-terminated; it is a view into the JSON.
|
* This is not null-terminated; it is a view into the JSON.
|
||||||
*
|
*
|
||||||
* @param index The position of the token to retrieve.
|
* @param position The position of the token to retrieve.
|
||||||
*
|
|
||||||
* TODO consider a string_view, assuming the length will get stripped out by the optimizer when
|
|
||||||
* it isn't used ...
|
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline const uint8_t *peek(token_position position) const noexcept;
|
simdjson_really_inline const uint8_t *peek(token_position position) const noexcept;
|
||||||
|
/**
|
||||||
|
* Get a pointer to the current location in the input buffer.
|
||||||
|
*
|
||||||
|
* This is not null-terminated; it is a view into the JSON.
|
||||||
|
*
|
||||||
|
* You may be pointing outside of the input buffer: it is not generally
|
||||||
|
* safe to derefence this pointer.
|
||||||
|
*/
|
||||||
|
simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept;
|
||||||
/**
|
/**
|
||||||
* Get the maximum length of the JSON text for the current token (or relative).
|
* Get the maximum length of the JSON text for the current token (or relative).
|
||||||
*
|
*
|
||||||
* The length will include any whitespace at the end of the token.
|
* The length will include any whitespace at the end of the token.
|
||||||
*
|
*
|
||||||
* @param index The position of the token to retrieve.
|
* @param position The position of the token to retrieve.
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline uint32_t peek_length(token_position position) const noexcept;
|
simdjson_really_inline uint32_t peek_length(token_position position) const noexcept;
|
||||||
/**
|
/**
|
||||||
@@ -218,8 +240,6 @@ public:
|
|||||||
*/
|
*/
|
||||||
simdjson_really_inline error_code optional_error(error_code error, const char *message) noexcept;
|
simdjson_really_inline error_code optional_error(error_code error, const char *message) noexcept;
|
||||||
|
|
||||||
template<int N> simdjson_warn_unused simdjson_really_inline bool copy_to_buffer(const uint8_t *json, uint32_t max_len, uint8_t (&tmpbuf)[N]) noexcept;
|
|
||||||
|
|
||||||
simdjson_really_inline token_position position() const noexcept;
|
simdjson_really_inline token_position position() const noexcept;
|
||||||
simdjson_really_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
|
simdjson_really_inline void reenter_child(token_position position, depth_t child_depth) noexcept;
|
||||||
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
||||||
@@ -238,9 +258,9 @@ protected:
|
|||||||
/// The last token before the end
|
/// The last token before the end
|
||||||
simdjson_really_inline token_position last_position() const noexcept;
|
simdjson_really_inline token_position last_position() const noexcept;
|
||||||
/// The token *at* the end. This points at gibberish and should only be used for comparison.
|
/// The token *at* the end. This points at gibberish and should only be used for comparison.
|
||||||
simdjson_really_inline token_position end_position() const noexcept;
|
simdjson_really_inline token_position end_of_input_buffer_position() const noexcept;
|
||||||
/// The end of the buffer.
|
/// The end of the buffer.
|
||||||
simdjson_really_inline token_position end() const noexcept;
|
simdjson_really_inline const uint8_t *end_of_input_buffer() const noexcept;
|
||||||
|
|
||||||
friend class document;
|
friend class document;
|
||||||
friend class document_stream;
|
friend class document_stream;
|
||||||
|
|||||||
@@ -124,6 +124,7 @@ inline void log_headers() noexcept {
|
|||||||
inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept {
|
inline void log_line(const json_iterator &iter, const char *title_prefix, const char *title, std::string_view detail, int delta, int depth_delta) noexcept {
|
||||||
log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail);
|
log_line(iter, iter.position()+delta, depth_t(iter.depth()+depth_delta), title_prefix, title, detail);
|
||||||
}
|
}
|
||||||
|
|
||||||
inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept {
|
inline void log_line(const json_iterator &iter, token_position index, depth_t depth, const char *title_prefix, const char *title, std::string_view detail) noexcept {
|
||||||
if (LOG_ENABLED) {
|
if (LOG_ENABLED) {
|
||||||
const int indent = depth*2;
|
const int indent = depth*2;
|
||||||
|
|||||||
@@ -42,7 +42,9 @@ simdjson_really_inline simdjson_result<object> object::start_root(value_iterator
|
|||||||
return object(iter);
|
return object(iter);
|
||||||
}
|
}
|
||||||
simdjson_really_inline error_code object::consume() noexcept {
|
simdjson_really_inline error_code object::consume() noexcept {
|
||||||
if(iter.is_at_key()) {
|
bool at_key;
|
||||||
|
SIMDJSON_TRY( iter.is_at_key().get(at_key) );
|
||||||
|
if(at_key) {
|
||||||
/**
|
/**
|
||||||
* whenever you are pointing at a key, calling skip_child() is
|
* whenever you are pointing at a key, calling skip_child() is
|
||||||
* unsafe because you will hit a string and you will assume that
|
* unsafe because you will hit a string and you will assume that
|
||||||
@@ -71,10 +73,12 @@ simdjson_really_inline simdjson_result<std::string_view> object::raw_json() noex
|
|||||||
const uint8_t * starting_point{iter.peek_start()};
|
const uint8_t * starting_point{iter.peek_start()};
|
||||||
auto error = consume();
|
auto error = consume();
|
||||||
if(error) { return error; }
|
if(error) { return error; }
|
||||||
const uint8_t * final_point{iter._json_iter->peek(0)};
|
// After 'consume()', we could be left pointing just beyond the document, but that
|
||||||
|
// is ok because we are not going to dereference the final pointer position, we just
|
||||||
|
// use it to compute the length in bytes.
|
||||||
|
const uint8_t * final_point{iter._json_iter->unsafe_pointer()};
|
||||||
return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
|
return std::string_view(reinterpret_cast<const char*>(starting_point), size_t(final_point - starting_point));
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline simdjson_result<object> object::started(value_iterator &iter) noexcept {
|
simdjson_really_inline simdjson_result<object> object::started(value_iterator &iter) noexcept {
|
||||||
SIMDJSON_TRY( iter.started_object().error() );
|
SIMDJSON_TRY( iter.started_object().error() );
|
||||||
return object(iter);
|
return object(iter);
|
||||||
|
|||||||
@@ -28,17 +28,25 @@ simdjson_warn_unused simdjson_really_inline error_code parser::allocate(size_t n
|
|||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(padded_string_view json) & noexcept {
|
|
||||||
if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
|
|
||||||
|
|
||||||
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const char *json, size_t len) & noexcept {
|
||||||
// Allocate if needed
|
// Allocate if needed
|
||||||
if (capacity() < json.length() || !string_buf) {
|
if (capacity() < len || !string_buf) {
|
||||||
SIMDJSON_TRY( allocate(json.length(), max_depth()) );
|
SIMDJSON_TRY( allocate(len, max_depth()) );
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run stage 1.
|
// Run stage 1.
|
||||||
SIMDJSON_TRY( implementation->stage1(reinterpret_cast<const uint8_t *>(json.data()), json.length(), stage1_mode::regular) );
|
SIMDJSON_TRY( implementation->stage1(reinterpret_cast<const uint8_t *>(json), len, stage1_mode::regular) );
|
||||||
return document::start({ reinterpret_cast<const uint8_t *>(json.data()), this });
|
return document::start({ reinterpret_cast<const uint8_t *>(json), this });
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(std::string_view view) & noexcept {
|
||||||
|
return iterate(view.data(), view.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(padded_string_view json) & noexcept {
|
||||||
|
// Iterate does not require any padding!!!
|
||||||
|
return iterate(json.data(), json.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const char *json, size_t len, size_t allocated) & noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const char *json, size_t len, size_t allocated) & noexcept {
|
||||||
@@ -54,7 +62,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::it
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const std::string &json) & noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const std::string &json) & noexcept {
|
||||||
return iterate(padded_string_view(json));
|
return iterate(json.data(), json.size());
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const simdjson_result<padded_string_view> &result) & noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const simdjson_result<padded_string_view> &result) & noexcept {
|
||||||
@@ -64,6 +72,11 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::it
|
|||||||
return iterate(json);
|
return iterate(json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const padded_string & json) & noexcept {
|
||||||
|
// We do not use the padding at all anymore.
|
||||||
|
return iterate(json.data(), json.size());
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const simdjson_result<padded_string> &result) & noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::iterate(const simdjson_result<padded_string> &result) & noexcept {
|
||||||
// We don't presently have a way to temporarily get a const T& from a simdjson_result<T> without throwing an exception
|
// We don't presently have a way to temporarily get a const T& from a simdjson_result<T> without throwing an exception
|
||||||
SIMDJSON_TRY( result.error() );
|
SIMDJSON_TRY( result.error() );
|
||||||
@@ -72,7 +85,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<document> parser::it
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<json_iterator> parser::iterate_raw(padded_string_view json) & noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<json_iterator> parser::iterate_raw(padded_string_view json) & noexcept {
|
||||||
if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
|
// Iterate does not require any padding!!!
|
||||||
|
// if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; }
|
||||||
|
|
||||||
// Allocate if needed
|
// Allocate if needed
|
||||||
if (capacity() < json.length()) {
|
if (capacity() < json.length()) {
|
||||||
@@ -97,6 +111,9 @@ inline simdjson_result<document_stream> parser::iterate_many(const std::string &
|
|||||||
inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
|
inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
|
||||||
return iterate_many(s.data(), s.length(), batch_size);
|
return iterate_many(s.data(), s.length(), batch_size);
|
||||||
}
|
}
|
||||||
|
inline simdjson_result<document_stream> parser::iterate_many(std::string_view view, size_t batch_size) noexcept {
|
||||||
|
return iterate_many(view.data(), view.length(), batch_size);
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_really_inline size_t parser::capacity() const noexcept {
|
simdjson_really_inline size_t parser::capacity() const noexcept {
|
||||||
return _capacity;
|
return _capacity;
|
||||||
|
|||||||
@@ -54,6 +54,8 @@ public:
|
|||||||
* ondemand::parser parser;
|
* ondemand::parser parser;
|
||||||
* document doc = parser.iterate(json);
|
* document doc = parser.iterate(json);
|
||||||
*
|
*
|
||||||
|
* The input buffer does not need to be padded.
|
||||||
|
*
|
||||||
* ### IMPORTANT: Validate what you use
|
* ### IMPORTANT: Validate what you use
|
||||||
*
|
*
|
||||||
* Calling iterate on an invalid JSON document may not immediately trigger an error. The call to
|
* Calling iterate on an invalid JSON document may not immediately trigger an error. The call to
|
||||||
@@ -70,17 +72,11 @@ public:
|
|||||||
* iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before
|
* iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before
|
||||||
* you call parse() again or destroy the parser.
|
* you call parse() again or destroy the parser.
|
||||||
*
|
*
|
||||||
* ### REQUIRED: Buffer Padding
|
|
||||||
*
|
|
||||||
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
|
|
||||||
* those bytes are initialized to, as long as they are allocated.
|
|
||||||
*
|
*
|
||||||
* @param json The JSON to parse.
|
* @param json The JSON to parse.
|
||||||
* @param len The length of the JSON.
|
* @param len The length of the JSON.
|
||||||
* @param capacity The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING).
|
|
||||||
*
|
*
|
||||||
* @return The document, or an error:
|
* @return The document, or an error:
|
||||||
* - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes.
|
|
||||||
* - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory
|
* - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory
|
||||||
* allocation fails.
|
* allocation fails.
|
||||||
* - EMPTY if the document is all whitespace.
|
* - EMPTY if the document is all whitespace.
|
||||||
@@ -88,61 +84,128 @@ public:
|
|||||||
* - UNESCAPED_CHARS if a string contains control characters that must be escaped
|
* - UNESCAPED_CHARS if a string contains control characters that must be escaped
|
||||||
* - UNCLOSED_STRING if there is an unclosed string in the document.
|
* - UNCLOSED_STRING if there is an unclosed string in the document.
|
||||||
*/
|
*/
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(padded_string_view json) & noexcept;
|
simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
/** @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept */
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len, size_t capacity) & noexcept;
|
simdjson_warn_unused simdjson_result<document> iterate(std::string_view view) & noexcept;
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
/** @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept */
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(const uint8_t *json, size_t len, size_t capacity) & noexcept;
|
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(std::string_view json, size_t capacity) & noexcept;
|
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(const std::string &json) & noexcept;
|
simdjson_warn_unused simdjson_result<document> iterate(const std::string &json) & noexcept;
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
|
simdjson_warn_unused simdjson_result<document> iterate(padded_string_view json) & noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
|
simdjson_warn_unused simdjson_result<document> iterate(const padded_string & json) & noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
|
simdjson_warn_unused simdjson_result<document> iterate(const char *json, size_t len, size_t capacity) & noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
|
simdjson_warn_unused simdjson_result<document> iterate(const uint8_t *json, size_t len, size_t capacity) & noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
|
simdjson_warn_unused simdjson_result<document> iterate(std::string_view json, size_t capacity) & noexcept;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string> &json) & noexcept;
|
simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string> &json) & noexcept;
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string_view> &json) & noexcept;
|
simdjson_warn_unused simdjson_result<document> iterate(const simdjson_result<padded_string_view> &json) & noexcept;
|
||||||
/** @overload simdjson_result<document> iterate(padded_string_view json) & noexcept */
|
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
simdjson_warn_unused simdjson_result<document> iterate(padded_string &&json) & noexcept = delete;
|
simdjson_warn_unused simdjson_result<document> iterate(padded_string &&json) & noexcept = delete;
|
||||||
|
/**
|
||||||
|
* @overload simdjson_result<document> iterate(const char *json, size_t len) & noexcept;
|
||||||
|
* iterate does not require padding, this function is kept for backward compatibility and convenience.
|
||||||
|
**/
|
||||||
|
simdjson_warn_unused simdjson_result<document> iterate(std::string &&json) & noexcept = delete;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @private
|
* @private
|
||||||
*
|
|
||||||
* Start iterating an on-demand JSON document.
|
|
||||||
*
|
|
||||||
* ondemand::parser parser;
|
|
||||||
* json_iterator doc = parser.iterate(json);
|
|
||||||
*
|
|
||||||
* ### IMPORTANT: Buffer Lifetime
|
|
||||||
*
|
|
||||||
* Because parsing is done while you iterate, you *must* keep the JSON buffer around at least as
|
|
||||||
* long as the document iteration.
|
|
||||||
*
|
|
||||||
* ### IMPORTANT: Document Lifetime
|
|
||||||
*
|
|
||||||
* Only one iteration at a time can happen per parser, and the parser *must* be kept alive during
|
|
||||||
* iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before
|
|
||||||
* you call parse() again or destroy the parser.
|
|
||||||
*
|
|
||||||
* ### REQUIRED: Buffer Padding
|
|
||||||
*
|
|
||||||
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
|
|
||||||
* those bytes are initialized to, as long as they are allocated.
|
|
||||||
*
|
|
||||||
* @param json The JSON to parse.
|
|
||||||
* @param len The length of the JSON.
|
|
||||||
* @param allocated The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING).
|
|
||||||
*
|
|
||||||
* @return The iterator, or an error:
|
|
||||||
* - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes.
|
|
||||||
* - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory
|
|
||||||
* allocation fails.
|
|
||||||
* - EMPTY if the document is all whitespace.
|
|
||||||
* - UTF8_ERROR if the document is not valid UTF-8.
|
|
||||||
* - UNESCAPED_CHARS if a string contains control characters that must be escaped
|
|
||||||
* - UNCLOSED_STRING if there is an unclosed string in the document.
|
|
||||||
*/
|
*/
|
||||||
simdjson_warn_unused simdjson_result<json_iterator> iterate_raw(padded_string_view json) & noexcept;
|
simdjson_warn_unused simdjson_result<json_iterator> iterate_raw(padded_string_view json) & noexcept;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse a buffer containing many JSON documents.
|
||||||
|
*
|
||||||
|
* std::string json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )";
|
||||||
|
* ondemand::parser parser;
|
||||||
|
* ondemand::document_stream docs = parser.iterate_many(json);
|
||||||
|
* for (auto & doc : docs) {
|
||||||
|
* std::cout << doc["foo"] << std::endl;
|
||||||
|
* }
|
||||||
|
* // Prints 1 2 3
|
||||||
|
*
|
||||||
|
* No copy of the input buffer is made.
|
||||||
|
*
|
||||||
|
* The function is lazy: it may be that no more than one JSON document at a time is parsed.
|
||||||
|
|
||||||
|
*
|
||||||
|
* The caller is responsabile to ensure that the input string data remains unchanged and is
|
||||||
|
* not deleted during the loop.
|
||||||
|
* ### Format
|
||||||
|
*
|
||||||
|
* The buffer must contain a series of one or more JSON documents, concatenated into a single
|
||||||
|
* buffer, separated by whitespace. It effectively parses until it has a fully valid document,
|
||||||
|
* then starts parsing the next document at that point. (It does this with more parallelism and
|
||||||
|
* lookahead than you might think, though.)
|
||||||
|
*
|
||||||
|
* documents that consist of an object or array may omit the whitespace between them, concatenating
|
||||||
|
* with no separator. documents that consist of a single primitive (i.e. documents that are not
|
||||||
|
* arrays or objects) MUST be separated with whitespace.
|
||||||
|
*
|
||||||
|
* The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse.
|
||||||
|
* Setting batch_size to excessively large or excesively small values may impact negatively the
|
||||||
|
* performance.
|
||||||
|
*
|
||||||
|
* ### Threads
|
||||||
|
*
|
||||||
|
* When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the
|
||||||
|
* hood to do some lookahead.
|
||||||
|
*
|
||||||
|
* ### Parser Capacity
|
||||||
|
*
|
||||||
|
* If the parser's current capacity is less than batch_size, it will allocate enough capacity
|
||||||
|
* to handle it (up to max_capacity).
|
||||||
|
*
|
||||||
|
* @param buf The concatenated JSON to parse.
|
||||||
|
* @param len The length of the concatenated JSON.
|
||||||
|
* @param batch_size The batch size to use. MUST be larger than the largest document. The sweet
|
||||||
|
* spot is cache-related: small enough to fit in cache, yet big enough to
|
||||||
|
* parse as many documents as possible in one tight loop.
|
||||||
|
* Defaults to 10MB, which has been a reasonable sweet spot in our tests.
|
||||||
|
* @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
|
||||||
|
* - MEMALLOC if the parser does not have enough capacity and memory allocation fails
|
||||||
|
* - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
|
||||||
|
* - other json errors if parsing fails. You should not rely on these errors to always the same for the
|
||||||
|
* same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
|
||||||
|
*/
|
||||||
|
inline simdjson_result<document_stream> iterate_many(std::string_view view, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
||||||
inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
||||||
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
|
||||||
inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
|
||||||
|
|||||||
@@ -6,8 +6,9 @@ namespace ondemand {
|
|||||||
simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) noexcept : buf{_buf} {}
|
simdjson_really_inline raw_json_string::raw_json_string(const uint8_t * _buf) noexcept : buf{_buf} {}
|
||||||
|
|
||||||
simdjson_really_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast<const char *>(buf); }
|
simdjson_really_inline const char * raw_json_string::raw() const noexcept { return reinterpret_cast<const char *>(buf); }
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst) const noexcept {
|
|
||||||
uint8_t *end = stringparsing::parse_string(buf, dst);
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept {
|
||||||
|
uint8_t *end = stringparsing::parse_string(buf, dst, buf_end);
|
||||||
if (!end) { return STRING_ERROR; }
|
if (!end) { return STRING_ERROR; }
|
||||||
std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
|
std::string_view result(reinterpret_cast<const char *>(dst), end-dst);
|
||||||
dst = end;
|
dst = end;
|
||||||
@@ -51,17 +52,19 @@ simdjson_really_inline bool raw_json_string::is_free_from_unescaped_quote(const
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
simdjson_really_inline bool raw_json_string::unsafe_is_equal(size_t length, std::string_view target) const noexcept {
|
simdjson_really_inline bool raw_json_string::unsafe_is_equal(size_t max_key_length_including_final_quote, std::string_view target) const noexcept {
|
||||||
// If we are going to call memcmp, then we must know something about the length of the raw_json_string.
|
// If we are going to call memcmp, then we must know something about the length of the raw_json_string.
|
||||||
return (length >= target.size()) && (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size());
|
if(max_key_length_including_final_quote <= target.size()) { return false; }
|
||||||
|
// It is now safe to read in [0, target.size()].
|
||||||
|
return (raw()[target.size()] == '"') && (memcmp(raw(), target.data(), target.size()) == 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept {
|
simdjson_really_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept {
|
||||||
// Assumptions: does not contain unescaped quote characters, and
|
// Assumptions: does not contain unescaped quote characters, and
|
||||||
// the raw content is quote terminated within a valid JSON string.
|
// the raw content is quote terminated within a valid JSON string.
|
||||||
if(target.size() <= SIMDJSON_PADDING) {
|
// if(target.size() <= SIMDJSON_PADDING) {
|
||||||
return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size());
|
// return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size());
|
||||||
}
|
// }
|
||||||
const char * r{raw()};
|
const char * r{raw()};
|
||||||
size_t pos{0};
|
size_t pos{0};
|
||||||
for(;pos < target.size();pos++) {
|
for(;pos < target.size();pos++) {
|
||||||
@@ -150,7 +153,7 @@ simdjson_unused simdjson_really_inline bool operator!=(std::string_view c, const
|
|||||||
|
|
||||||
|
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> raw_json_string::unescape(json_iterator &iter) const noexcept {
|
||||||
return unescape(iter.string_buf_loc());
|
return unescape(iter.string_buf_loc(), iter.end_of_input_buffer());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -183,9 +186,9 @@ simdjson_really_inline simdjson_result<const char *> simdjson_result<SIMDJSON_IM
|
|||||||
if (error()) { return error(); }
|
if (error()) { return error(); }
|
||||||
return first.raw();
|
return first.raw();
|
||||||
}
|
}
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(uint8_t *&dst) const noexcept {
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept {
|
||||||
if (error()) { return error(); }
|
if (error()) { return error(); }
|
||||||
return first.unescape(dst);
|
return first.unescape(dst, buf_end);
|
||||||
}
|
}
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string>::unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept {
|
||||||
if (error()) { return error(); }
|
if (error()) { return error(); }
|
||||||
|
|||||||
@@ -53,8 +53,11 @@ public:
|
|||||||
* This compares the current instance to the std::string_view target: returns true if
|
* This compares the current instance to the std::string_view target: returns true if
|
||||||
* they are byte-by-byte equal (no escaping is done) on target.size() characters,
|
* they are byte-by-byte equal (no escaping is done) on target.size() characters,
|
||||||
* and if the raw_json_string instance has a quote character at byte index target.size().
|
* and if the raw_json_string instance has a quote character at byte index target.size().
|
||||||
* We never read more than length + 1 bytes in the raw_json_string instance.
|
* We never read more than max_key_length_including_final_quote bytes in the raw_json_string instance.
|
||||||
* If length is smaller than target.size(), this will return false.
|
* If max_key_length_including_final_quote is smaller than target.size() + 1, this will return false.
|
||||||
|
*
|
||||||
|
* max_key_length_including_final_quote is the maximal key length in bytes, not including
|
||||||
|
* the leading quote.
|
||||||
*
|
*
|
||||||
* The std::string_view instance may contain any characters. However, the caller
|
* The std::string_view instance may contain any characters. However, the caller
|
||||||
* is responsible for setting length so that length bytes may be read in the
|
* is responsible for setting length so that length bytes may be read in the
|
||||||
@@ -63,7 +66,7 @@ public:
|
|||||||
* Performance: the comparison may be done using memcmp which may be efficient
|
* Performance: the comparison may be done using memcmp which may be efficient
|
||||||
* for long strings.
|
* for long strings.
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline bool unsafe_is_equal(size_t length, std::string_view target) const noexcept;
|
simdjson_really_inline bool unsafe_is_equal(size_t max_key_length_including_final_quote, std::string_view target) const noexcept;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This compares the current instance to the std::string_view target: returns true if
|
* This compares the current instance to the std::string_view target: returns true if
|
||||||
@@ -140,10 +143,13 @@ private:
|
|||||||
* @param dst A pointer to a buffer at least large enough to write this string as well as a \0.
|
* @param dst A pointer to a buffer at least large enough to write this string as well as a \0.
|
||||||
* dst will be updated to the next unused location (just after the \0 written out at
|
* dst will be updated to the next unused location (just after the \0 written out at
|
||||||
* the end of this string).
|
* the end of this string).
|
||||||
|
* @param buf_end A pointer to the end of the input JSON you passed to iterate(). This is
|
||||||
|
* used to prevent overruns, since simdjson normally reads (but does not use) past
|
||||||
|
* the end quote when parsing a string for performance reasons.
|
||||||
* @return A string_view pointing at the unescaped string in dst
|
* @return A string_view pointing at the unescaped string in dst
|
||||||
* @error STRING_ERROR if escapes are incorrect.
|
* @error STRING_ERROR if escapes are incorrect.
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept;
|
||||||
/**
|
/**
|
||||||
* Unescape this JSON string, replacing \\ with \, \n with newline, etc.
|
* Unescape this JSON string, replacing \\ with \, \n with newline, etc.
|
||||||
*
|
*
|
||||||
@@ -188,7 +194,7 @@ public:
|
|||||||
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
|
simdjson_really_inline ~simdjson_result() noexcept = default; ///< @private
|
||||||
|
|
||||||
simdjson_really_inline simdjson_result<const char *> raw() const noexcept;
|
simdjson_really_inline simdjson_result<const char *> raw() const noexcept;
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst) const noexcept;
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(uint8_t *&dst, const uint8_t *buf_end) const noexcept;
|
||||||
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
|
simdjson_really_inline simdjson_warn_unused simdjson_result<std::string_view> unescape(SIMDJSON_IMPLEMENTATION::ondemand::json_iterator &iter) const noexcept;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -295,9 +295,9 @@ public:
|
|||||||
* The string_view will always point into the input buffer.
|
* The string_view will always point into the input buffer.
|
||||||
*
|
*
|
||||||
* The string_view will start at the beginning of the token, and include the entire token
|
* The string_view will start at the beginning of the token, and include the entire token
|
||||||
* *as well as all spaces until the next token (or EOF).* This means, for example, that a
|
* *as well as all spaces until the next token (or end of the buffer, or EOF).*
|
||||||
* string token always begins with a " and is always terminated by the final ", possibly
|
* This means, for example, that a string token always begins with a " and is always
|
||||||
* followed by a number of spaces.
|
* terminated by the final ", possibly followed by a number of spaces.
|
||||||
*
|
*
|
||||||
* The string_view is *not* null-terminated. However, if this is a scalar (string, number,
|
* The string_view is *not* null-terminated. However, if this is a scalar (string, number,
|
||||||
* boolean, or null), the character after the end of the string_view is guaranteed to be
|
* boolean, or null), the character after the end of the string_view is guaranteed to be
|
||||||
|
|||||||
@@ -25,12 +25,15 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
||||||
_json_iter->set_start_position(_depth, start_position());
|
_json_iter->set_start_position(_depth, start_position());
|
||||||
#endif
|
#endif
|
||||||
|
SIMDJSON_TRY( _json_iter->require_tokens(1) );
|
||||||
if (*_json_iter->peek() == '}') {
|
if (*_json_iter->peek() == '}') {
|
||||||
logger::log_value(*_json_iter, "empty object");
|
logger::log_value(*_json_iter, "empty object");
|
||||||
_json_iter->return_current_and_advance();
|
_json_iter->return_current_and_advance();
|
||||||
end_container();
|
end_container();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
SIMDJSON_TRY( _json_iter->require_tokens(3) ); // Make sure we have three tokens: "x" : value
|
||||||
|
logger::log_start_value(*_json_iter, "object");
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,10 +49,10 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_container() noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_container() noexcept {
|
||||||
#if __SIMDJSON_CHECK_EOF
|
#if SIMDJSON_CHECK_EOF
|
||||||
if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); }
|
if (depth() > 1 && at_end_of_input_buffer()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); }
|
||||||
// if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); }
|
// if (depth() <= 1 && !at_end_of_input_buffer()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); }
|
||||||
#endif // __SIMDJSON_CHECK_EOF
|
#endif // SIMDJSON_CHECK_EOF
|
||||||
_json_iter->ascend_to(depth()-1);
|
_json_iter->ascend_to(depth()-1);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
@@ -57,6 +60,8 @@ simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_conta
|
|||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_field() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_field() noexcept {
|
||||||
assert_at_next();
|
assert_at_next();
|
||||||
|
|
||||||
|
SIMDJSON_TRY( error_unless_more_tokens() );
|
||||||
|
|
||||||
// It's illegal to call this unless there are more tokens: anything that ends in } or ] is
|
// It's illegal to call this unless there are more tokens: anything that ends in } or ] is
|
||||||
// obligated to verify there are more tokens if they are not the top level.
|
// obligated to verify there are more tokens if they are not the top level.
|
||||||
switch (*_json_iter->return_current_and_advance()) {
|
switch (*_json_iter->return_current_and_advance()) {
|
||||||
@@ -65,6 +70,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
SIMDJSON_TRY( end_container() );
|
SIMDJSON_TRY( end_container() );
|
||||||
return false;
|
return false;
|
||||||
case ',':
|
case ',':
|
||||||
|
SIMDJSON_TRY( error_unless_more_tokens(3) ); // Make sure we have three tokens: "x" : value
|
||||||
return true;
|
return true;
|
||||||
default:
|
default:
|
||||||
return report_error(TAPE_ERROR, "Missing comma between object fields");
|
return report_error(TAPE_ERROR, "Missing comma between object fields");
|
||||||
@@ -129,7 +135,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
while (has_value) {
|
while (has_value) {
|
||||||
// Get the key and colon, stopping at the value.
|
// Get the key and colon, stopping at the value.
|
||||||
raw_json_string actual_key;
|
raw_json_string actual_key;
|
||||||
// size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
|
// We know, for sure, that _json_iter->peek_length() is at least '1', but it could
|
||||||
|
// be just one, so subtracting by more than 1 is unsafe.
|
||||||
|
size_t max_key_length_including_final_quote = _json_iter->peek_length() - 1; // -1 for one quote
|
||||||
// field_key() advances the pointer and checks that '"' is found (corresponding to a key).
|
// field_key() advances the pointer and checks that '"' is found (corresponding to a key).
|
||||||
// The depth is left unchanged by field_key().
|
// The depth is left unchanged by field_key().
|
||||||
if ((error = field_key().get(actual_key) )) { abandon(); return error; };
|
if ((error = field_key().get(actual_key) )) { abandon(); return error; };
|
||||||
@@ -137,14 +145,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
// key and the value. It will also increment the depth by one.
|
// key and the value. It will also increment the depth by one.
|
||||||
if ((error = field_value() )) { abandon(); return error; }
|
if ((error = field_value() )) { abandon(); return error; }
|
||||||
// If it matches, stop and return
|
// If it matches, stop and return
|
||||||
// We could do it this way if we wanted to allow arbitrary
|
if (actual_key.unsafe_is_equal(max_key_length_including_final_quote, key)) {
|
||||||
// key content (including escaped quotes).
|
|
||||||
//if (actual_key.unsafe_is_equal(max_key_length, key)) {
|
|
||||||
// Instead we do the following which may trigger buffer overruns if the
|
|
||||||
// user provides an adversarial key (containing a well placed unescaped quote
|
|
||||||
// character and being longer than the number of bytes remaining in the JSON
|
|
||||||
// input).
|
|
||||||
if (actual_key.unsafe_is_equal(key)) {
|
|
||||||
logger::log_event(*this, "match", key, -2);
|
logger::log_event(*this, "match", key, -2);
|
||||||
// If we return here, then we return while pointing at the ':' that we just checked.
|
// If we return here, then we return while pointing at the ':' that we just checked.
|
||||||
return true;
|
return true;
|
||||||
@@ -259,7 +260,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
|
|
||||||
// Get the key and colon, stopping at the value.
|
// Get the key and colon, stopping at the value.
|
||||||
raw_json_string actual_key;
|
raw_json_string actual_key;
|
||||||
// size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
|
// We know, for sure, that _json_iter->peek_length() is at least '1', but it could
|
||||||
|
// be just one, so subtracting by more than 1 is unsafe.
|
||||||
|
size_t max_key_length_including_final_quote = _json_iter->peek_length() - 1; // -1 for one quote
|
||||||
// field_key() advances the pointer and checks that '"' is found (corresponding to a key).
|
// field_key() advances the pointer and checks that '"' is found (corresponding to a key).
|
||||||
// The depth is left unchanged by field_key().
|
// The depth is left unchanged by field_key().
|
||||||
if ((error = field_key().get(actual_key) )) { abandon(); return error; };
|
if ((error = field_key().get(actual_key) )) { abandon(); return error; };
|
||||||
@@ -268,14 +271,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
if ((error = field_value() )) { abandon(); return error; }
|
if ((error = field_value() )) { abandon(); return error; }
|
||||||
|
|
||||||
// If it matches, stop and return
|
// If it matches, stop and return
|
||||||
// We could do it this way if we wanted to allow arbitrary
|
if (actual_key.unsafe_is_equal(max_key_length_including_final_quote, key)) {
|
||||||
// key content (including escaped quotes).
|
|
||||||
// if (actual_key.unsafe_is_equal(max_key_length, key)) {
|
|
||||||
// Instead we do the following which may trigger buffer overruns if the
|
|
||||||
// user provides an adversarial key (containing a well placed unescaped quote
|
|
||||||
// character and being longer than the number of bytes remaining in the JSON
|
|
||||||
// input).
|
|
||||||
if (actual_key.unsafe_is_equal(key)) {
|
|
||||||
logger::log_event(*this, "match", key, -2);
|
logger::log_event(*this, "match", key, -2);
|
||||||
// If we return here, then we return while pointing at the ':' that we just checked.
|
// If we return here, then we return while pointing at the ':' that we just checked.
|
||||||
return true;
|
return true;
|
||||||
@@ -308,7 +304,9 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
|
|
||||||
// Get the key and colon, stopping at the value.
|
// Get the key and colon, stopping at the value.
|
||||||
raw_json_string actual_key;
|
raw_json_string actual_key;
|
||||||
// size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes
|
// We know, for sure, that _json_iter->peek_length() is at least '1', but it could
|
||||||
|
// be just one, so subtracting by more than 1 is unsafe.
|
||||||
|
size_t max_key_length_including_final_quote = _json_iter->peek_length() - 1; // -1 for one quote
|
||||||
// field_key() advances the pointer and checks that '"' is found (corresponding to a key).
|
// field_key() advances the pointer and checks that '"' is found (corresponding to a key).
|
||||||
// The depth is left unchanged by field_key().
|
// The depth is left unchanged by field_key().
|
||||||
error = field_key().get(actual_key); SIMDJSON_ASSUME(!error);
|
error = field_key().get(actual_key); SIMDJSON_ASSUME(!error);
|
||||||
@@ -317,14 +315,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
error = field_value(); SIMDJSON_ASSUME(!error);
|
error = field_value(); SIMDJSON_ASSUME(!error);
|
||||||
|
|
||||||
// If it matches, stop and return
|
// If it matches, stop and return
|
||||||
// We could do it this way if we wanted to allow arbitrary
|
if (actual_key.unsafe_is_equal(max_key_length_including_final_quote, key)) {
|
||||||
// key content (including escaped quotes).
|
|
||||||
// if (actual_key.unsafe_is_equal(max_key_length, key)) {
|
|
||||||
// Instead we do the following which may trigger buffer overruns if the
|
|
||||||
// user provides an adversarial key (containing a well placed unescaped quote
|
|
||||||
// character and being longer than the number of bytes remaining in the JSON
|
|
||||||
// input).
|
|
||||||
if (actual_key.unsafe_is_equal(key)) {
|
|
||||||
logger::log_event(*this, "match", key, -2);
|
logger::log_event(*this, "match", key, -2);
|
||||||
// If we return here, then we return while pointing at the ':' that we just checked.
|
// If we return here, then we return while pointing at the ':' that we just checked.
|
||||||
return true;
|
return true;
|
||||||
@@ -354,6 +345,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::field_key() noexcept {
|
||||||
assert_at_next();
|
assert_at_next();
|
||||||
|
|
||||||
|
// started_object() and has_next_field() already checked that we have a key
|
||||||
const uint8_t *key = _json_iter->return_current_and_advance();
|
const uint8_t *key = _json_iter->return_current_and_advance();
|
||||||
if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); }
|
if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); }
|
||||||
return raw_json_string(key);
|
return raw_json_string(key);
|
||||||
@@ -362,6 +354,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> val
|
|||||||
simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_value() noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code value_iterator::field_value() noexcept {
|
||||||
assert_at_next();
|
assert_at_next();
|
||||||
|
|
||||||
|
// started_object() and has_next_field() already checked that we have a : and a value token
|
||||||
if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
|
if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); }
|
||||||
_json_iter->descend_to(depth()+1);
|
_json_iter->descend_to(depth()+1);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
@@ -386,6 +379,8 @@ inline std::string value_iterator::to_string() const noexcept {
|
|||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::started_array() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::started_array() noexcept {
|
||||||
assert_at_container_start();
|
assert_at_container_start();
|
||||||
|
SIMDJSON_TRY( _json_iter->require_tokens(1) );
|
||||||
|
|
||||||
if (*_json_iter->peek() == ']') {
|
if (*_json_iter->peek() == ']') {
|
||||||
logger::log_value(*_json_iter, "empty array");
|
logger::log_value(*_json_iter, "empty array");
|
||||||
_json_iter->return_current_and_advance();
|
_json_iter->return_current_and_advance();
|
||||||
@@ -413,13 +408,16 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_element() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::has_next_element() noexcept {
|
||||||
assert_at_next();
|
assert_at_next();
|
||||||
|
|
||||||
|
const uint8_t *json;
|
||||||
logger::log_event(*this, "has_next_element");
|
logger::log_event(*this, "has_next_element");
|
||||||
switch (*_json_iter->return_current_and_advance()) {
|
SIMDJSON_TRY( _json_iter->try_return_current_and_advance().get(json) )
|
||||||
|
switch (*json) {
|
||||||
case ']':
|
case ']':
|
||||||
logger::log_end_value(*_json_iter, "array");
|
logger::log_end_value(*_json_iter, "array");
|
||||||
SIMDJSON_TRY( end_container() );
|
SIMDJSON_TRY( end_container() );
|
||||||
return false;
|
return false;
|
||||||
case ',':
|
case ',':
|
||||||
|
SIMDJSON_TRY( _json_iter->require_tokens(1) );
|
||||||
_json_iter->descend_to(depth()+1);
|
_json_iter->descend_to(depth()+1);
|
||||||
return true;
|
return true;
|
||||||
default:
|
default:
|
||||||
@@ -427,19 +425,8 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::parse_bool(const uint8_t *json) const noexcept {
|
|
||||||
auto not_true = atomparsing::str4ncmp(json, "true");
|
|
||||||
auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e');
|
|
||||||
bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]);
|
|
||||||
if (error) { return incorrect_type_error("Not a boolean"); }
|
|
||||||
return simdjson_result<bool>(!not_true);
|
|
||||||
}
|
|
||||||
simdjson_really_inline bool value_iterator::parse_null(const uint8_t *json) const noexcept {
|
|
||||||
return !atomparsing::str4ncmp(json, "null") && jsoncharutils::is_structural_or_whitespace(json[4]);
|
|
||||||
}
|
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_string() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_string() noexcept {
|
||||||
return get_raw_json_string().unescape(_json_iter->string_buf_loc());
|
return get_raw_json_string().unescape(*_json_iter);
|
||||||
}
|
}
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
|
||||||
auto json = peek_scalar("string");
|
auto json = peek_scalar("string");
|
||||||
@@ -463,16 +450,42 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterat
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::get_bool() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::get_bool() noexcept {
|
||||||
auto result = parse_bool(peek_non_root_scalar("bool"));
|
auto result = parse_bool(peek_start_length(), peek_non_root_scalar("bool"));
|
||||||
if(result.error() != INCORRECT_TYPE) { advance_non_root_scalar("bool"); }
|
if(result.error() != INCORRECT_TYPE) { advance_non_root_scalar("bool"); }
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline simdjson_result<bool> value_iterator::parse_bool(uint32_t max_len, const uint8_t *json) const noexcept {
|
||||||
|
if (max_len >= 4) {
|
||||||
|
if (!atomparsing::str4ncmp_unsafe(json, "true")) {
|
||||||
|
if (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4])) {
|
||||||
|
return simdjson_result<bool>(true);
|
||||||
|
}
|
||||||
|
} else if (!atomparsing::str4ncmp_unsafe(json, "fals") && max_len >= 5 && json[4] == 'e') {
|
||||||
|
if (max_len == 5 || jsoncharutils::is_structural_or_whitespace(json[5])) {
|
||||||
|
return simdjson_result<bool>(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return incorrect_type_error("Not a boolean");
|
||||||
|
}
|
||||||
simdjson_really_inline bool value_iterator::is_null() noexcept {
|
simdjson_really_inline bool value_iterator::is_null() noexcept {
|
||||||
auto result = parse_null(peek_non_root_scalar("null"));
|
auto result = parse_null(peek_start_length(), peek_non_root_scalar("null"));
|
||||||
if(result) { advance_non_root_scalar("null"); }
|
if(result) { advance_non_root_scalar("null"); }
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline bool value_iterator::parse_null(uint32_t max_len, const uint8_t *json) const noexcept {
|
||||||
|
if (max_len >= 4) {
|
||||||
|
if (!atomparsing::str4ncmp_unsafe(json, "null")) {
|
||||||
|
if (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4])) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
constexpr const uint32_t MAX_INT_LENGTH = 1024;
|
constexpr const uint32_t MAX_INT_LENGTH = 1024;
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_root_string() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<std::string_view> value_iterator::get_root_string() noexcept {
|
||||||
@@ -484,56 +497,31 @@ simdjson_warn_unused simdjson_really_inline simdjson_result<raw_json_string> val
|
|||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::get_root_uint64() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<uint64_t> value_iterator::get_root_uint64() noexcept {
|
||||||
auto max_len = peek_start_length();
|
auto max_len = peek_start_length();
|
||||||
auto json = peek_root_scalar("uint64");
|
auto json = peek_root_scalar("uint64");
|
||||||
uint8_t tmpbuf[20+1]; // <20 digits> is the longest possible unsigned integer
|
auto result = numberparsing::parse_unsigned(json, json+max_len);
|
||||||
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
|
|
||||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters");
|
|
||||||
return NUMBER_ERROR;
|
|
||||||
}
|
|
||||||
auto result = numberparsing::parse_unsigned(tmpbuf);
|
|
||||||
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("uint64"); }
|
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("uint64"); }
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::get_root_int64() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<int64_t> value_iterator::get_root_int64() noexcept {
|
||||||
auto max_len = peek_start_length();
|
auto max_len = peek_start_length();
|
||||||
auto json = peek_root_scalar("int64");
|
auto json = peek_root_scalar("int64");
|
||||||
uint8_t tmpbuf[20+1]; // -<19 digits> is the longest possible integer
|
auto result = numberparsing::parse_integer(json, json+max_len);
|
||||||
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
|
|
||||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters");
|
|
||||||
return NUMBER_ERROR;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto result = numberparsing::parse_integer(tmpbuf);
|
|
||||||
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("int64"); }
|
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("int64"); }
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::get_root_double() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<double> value_iterator::get_root_double() noexcept {
|
||||||
auto max_len = peek_start_length();
|
auto max_len = peek_start_length();
|
||||||
auto json = peek_root_scalar("double");
|
auto json = peek_root_scalar("double");
|
||||||
// Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/,
|
auto result = numberparsing::parse_double(json, json+max_len);
|
||||||
// 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest
|
|
||||||
// number: -0.<fraction>e-308.
|
|
||||||
uint8_t tmpbuf[1074+8+1];
|
|
||||||
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) {
|
|
||||||
logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters");
|
|
||||||
return NUMBER_ERROR;
|
|
||||||
}
|
|
||||||
auto result = numberparsing::parse_double(tmpbuf);
|
|
||||||
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("double"); }
|
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("double"); }
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::get_root_bool() noexcept {
|
simdjson_warn_unused simdjson_really_inline simdjson_result<bool> value_iterator::get_root_bool() noexcept {
|
||||||
auto max_len = peek_start_length();
|
auto result = parse_bool(peek_start_length(), peek_root_scalar("bool"));
|
||||||
auto json = peek_root_scalar("bool");
|
if(result.error() != INCORRECT_TYPE) { advance_root_scalar("bool"); }
|
||||||
uint8_t tmpbuf[5+1];
|
return result;
|
||||||
if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf)) { return incorrect_type_error("Not a boolean"); }
|
|
||||||
advance_root_scalar("bool");
|
|
||||||
return parse_bool(tmpbuf);
|
|
||||||
}
|
}
|
||||||
simdjson_really_inline bool value_iterator::is_root_null() noexcept {
|
simdjson_really_inline bool value_iterator::is_root_null() noexcept {
|
||||||
auto max_len = peek_start_length();
|
auto result = parse_null(peek_start_length(), peek_root_scalar("null"));
|
||||||
auto json = peek_root_scalar("null");
|
|
||||||
auto result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") &&
|
|
||||||
(max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[5])));
|
|
||||||
if(result) { advance_root_scalar("null"); }
|
if(result) { advance_root_scalar("null"); }
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
@@ -560,8 +548,8 @@ simdjson_really_inline bool value_iterator::is_open() const noexcept {
|
|||||||
}
|
}
|
||||||
SIMDJSON_POP_DISABLE_WARNINGS
|
SIMDJSON_POP_DISABLE_WARNINGS
|
||||||
|
|
||||||
simdjson_really_inline bool value_iterator::at_end() const noexcept {
|
simdjson_really_inline bool value_iterator::at_end_of_input_buffer() const noexcept {
|
||||||
return _json_iter->at_end();
|
return _json_iter->at_end_of_input_buffer();
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool value_iterator::at_start() const noexcept {
|
simdjson_really_inline bool value_iterator::at_start() const noexcept {
|
||||||
@@ -594,10 +582,12 @@ simdjson_warn_unused simdjson_really_inline json_iterator &value_iterator::json_
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline const uint8_t *value_iterator::peek_start() const noexcept {
|
simdjson_really_inline const uint8_t *value_iterator::peek_start() const noexcept {
|
||||||
return _json_iter->peek(start_position());
|
auto result = _json_iter->peek(start_position());
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
simdjson_really_inline uint32_t value_iterator::peek_start_length() const noexcept {
|
simdjson_really_inline uint32_t value_iterator::peek_start_length() const noexcept {
|
||||||
return _json_iter->peek_length(start_position());
|
auto result = _json_iter->peek_length(start_position());
|
||||||
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept {
|
simdjson_really_inline const uint8_t *value_iterator::peek_scalar(const char *type) noexcept {
|
||||||
@@ -685,15 +675,29 @@ simdjson_really_inline error_code value_iterator::incorrect_type_error(const cha
|
|||||||
return INCORRECT_TYPE;
|
return INCORRECT_TYPE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline error_code value_iterator::error_unless_more_tokens(uint32_t tokens) const noexcept {
|
||||||
|
if ((position() + tokens) > end_of_input_buffer_position()) {
|
||||||
|
return _json_iter->report_error(TAPE_ERROR, "Document ended early");
|
||||||
|
}
|
||||||
|
return SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool value_iterator::is_at_start() const noexcept {
|
simdjson_really_inline bool value_iterator::is_at_start() const noexcept {
|
||||||
return position() == start_position();
|
return position() == start_position();
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool value_iterator::is_at_key() const noexcept {
|
simdjson_really_inline simdjson_result<bool> value_iterator::is_at_key() const noexcept {
|
||||||
// Keys are at the same depth as the object.
|
// Keys are at the same depth as the object.
|
||||||
// Note here that we could be safer and check that we are within an object,
|
// Note here that we could be safer and check that we are within an object,
|
||||||
// but we do not.
|
// but we do not.
|
||||||
return _depth == _json_iter->_depth && *_json_iter->peek() == '"';
|
if(_depth == _json_iter->_depth) {
|
||||||
|
// If it is at a key, we need 3 more tokens, but if it is not at a key, then a single
|
||||||
|
// token would suffice (i.e., '}').
|
||||||
|
SIMDJSON_TRY( _json_iter->require_tokens(1) );
|
||||||
|
|
||||||
|
return *_json_iter->peek() == '"';
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool value_iterator::is_at_iterator_start() const noexcept {
|
simdjson_really_inline bool value_iterator::is_at_iterator_start() const noexcept {
|
||||||
@@ -793,8 +797,8 @@ simdjson_really_inline token_position value_iterator::position() const noexcept
|
|||||||
return _json_iter->position();
|
return _json_iter->position();
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline token_position value_iterator::end_position() const noexcept {
|
simdjson_really_inline token_position value_iterator::end_of_input_buffer_position() const noexcept {
|
||||||
return _json_iter->end_position();
|
return _json_iter->end_of_input_buffer_position();
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline token_position value_iterator::last_position() const noexcept {
|
simdjson_really_inline token_position value_iterator::last_position() const noexcept {
|
||||||
|
|||||||
@@ -44,9 +44,9 @@ public:
|
|||||||
simdjson_warn_unused simdjson_really_inline error_code skip_child() noexcept;
|
simdjson_warn_unused simdjson_really_inline error_code skip_child() noexcept;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tell whether the iterator is at the EOF mark
|
* Tell whether the iterator is at the EOF mark (end of input buffer)
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline bool at_end() const noexcept;
|
simdjson_really_inline bool at_end_of_input_buffer() const noexcept;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tell whether the iterator is at the start of the value
|
* Tell whether the iterator is at the start of the value
|
||||||
@@ -336,8 +336,8 @@ protected:
|
|||||||
inline std::string to_string() const noexcept;
|
inline std::string to_string() const noexcept;
|
||||||
simdjson_really_inline value_iterator(json_iterator *json_iter, depth_t depth, token_position start_index) noexcept;
|
simdjson_really_inline value_iterator(json_iterator *json_iter, depth_t depth, token_position start_index) noexcept;
|
||||||
|
|
||||||
simdjson_really_inline bool parse_null(const uint8_t *json) const noexcept;
|
simdjson_really_inline bool parse_null(uint32_t max_len, const uint8_t *json) const noexcept;
|
||||||
simdjson_really_inline simdjson_result<bool> parse_bool(const uint8_t *json) const noexcept;
|
simdjson_really_inline simdjson_result<bool> parse_bool(uint32_t max_len, const uint8_t *json) const noexcept;
|
||||||
simdjson_really_inline const uint8_t *peek_start() const noexcept;
|
simdjson_really_inline const uint8_t *peek_start() const noexcept;
|
||||||
simdjson_really_inline uint32_t peek_start_length() const noexcept;
|
simdjson_really_inline uint32_t peek_start_length() const noexcept;
|
||||||
|
|
||||||
@@ -409,7 +409,7 @@ protected:
|
|||||||
* Usage: the skip_child() method should never be used while we are pointing
|
* Usage: the skip_child() method should never be used while we are pointing
|
||||||
* at a key inside an object.
|
* at a key inside an object.
|
||||||
*/
|
*/
|
||||||
simdjson_really_inline bool is_at_key() const noexcept;
|
simdjson_really_inline simdjson_result<bool> is_at_key() const noexcept;
|
||||||
|
|
||||||
inline void assert_at_start() const noexcept;
|
inline void assert_at_start() const noexcept;
|
||||||
inline void assert_at_container_start() const noexcept;
|
inline void assert_at_container_start() const noexcept;
|
||||||
@@ -423,10 +423,10 @@ protected:
|
|||||||
|
|
||||||
/** @copydoc error_code json_iterator::position() const noexcept; */
|
/** @copydoc error_code json_iterator::position() const noexcept; */
|
||||||
simdjson_really_inline token_position position() const noexcept;
|
simdjson_really_inline token_position position() const noexcept;
|
||||||
/** @copydoc error_code json_iterator::end_position() const noexcept; */
|
/** @copydoc error_code json_iterator::end_of_input_buffer_position() const noexcept; */
|
||||||
simdjson_really_inline token_position last_position() const noexcept;
|
simdjson_really_inline token_position last_position() const noexcept;
|
||||||
/** @copydoc error_code json_iterator::end_position() const noexcept; */
|
/** @copydoc error_code json_iterator::end_of_input_buffer_position() const noexcept; */
|
||||||
simdjson_really_inline token_position end_position() const noexcept;
|
simdjson_really_inline token_position end_of_input_buffer_position() const noexcept;
|
||||||
/** @copydoc error_code json_iterator::report_error(error_code error, const char *message) noexcept; */
|
/** @copydoc error_code json_iterator::report_error(error_code error, const char *message) noexcept; */
|
||||||
simdjson_really_inline error_code report_error(error_code error, const char *message) noexcept;
|
simdjson_really_inline error_code report_error(error_code error, const char *message) noexcept;
|
||||||
|
|
||||||
@@ -434,6 +434,7 @@ protected:
|
|||||||
friend class object;
|
friend class object;
|
||||||
friend class array;
|
friend class array;
|
||||||
friend class value;
|
friend class value;
|
||||||
|
friend class field;
|
||||||
}; // value_iterator
|
}; // value_iterator
|
||||||
|
|
||||||
} // namespace ondemand
|
} // namespace ondemand
|
||||||
|
|||||||
@@ -77,10 +77,11 @@ simdjson_really_inline bool handle_unicode_codepoint(const uint8_t **src_ptr,
|
|||||||
* Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then
|
* Unescape a string from src to dst, stopping at a final unescaped quote. E.g., if src points at 'joe"', then
|
||||||
* dst needs to have four free bytes.
|
* dst needs to have four free bytes.
|
||||||
*/
|
*/
|
||||||
simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst) {
|
simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t *src, uint8_t *dst, const uint8_t *end) {
|
||||||
|
const uint8_t *last_full_buf = end - backslash_and_quote::BYTES_PROCESSED;
|
||||||
while (1) {
|
while (1) {
|
||||||
// Copy the next n bytes, and find the backslash and quote in them.
|
// Copy the next n bytes, and find the backslash and quote in them.
|
||||||
auto bs_quote = backslash_and_quote::copy_and_find(src, dst);
|
auto bs_quote = backslash_and_quote::copy_and_find(src, dst, last_full_buf);
|
||||||
// If the next thing is the end quote, copy and return
|
// If the next thing is the end quote, copy and return
|
||||||
if (bs_quote.has_quote_first()) {
|
if (bs_quote.has_quote_first()) {
|
||||||
// we encountered quotes first. Move dst to point to quotes and exit
|
// we encountered quotes first. Move dst to point to quotes and exit
|
||||||
@@ -123,15 +124,6 @@ simdjson_warn_unused simdjson_really_inline uint8_t *parse_string(const uint8_t
|
|||||||
return nullptr;
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_unused simdjson_warn_unused simdjson_really_inline error_code parse_string_to_buffer(const uint8_t *src, uint8_t *¤t_string_buf_loc, std::string_view &s) {
|
|
||||||
if (*(src++) != '"') { return STRING_ERROR; }
|
|
||||||
auto end = stringparsing::parse_string(src, current_string_buf_loc);
|
|
||||||
if (!end) { return STRING_ERROR; }
|
|
||||||
s = std::string_view(reinterpret_cast<const char *>(current_string_buf_loc), end-current_string_buf_loc);
|
|
||||||
current_string_buf_loc = end;
|
|
||||||
return SUCCESS;
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace stringparsing
|
} // namespace stringparsing
|
||||||
} // unnamed namespace
|
} // unnamed namespace
|
||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
|
|||||||
@@ -5,6 +5,33 @@ namespace simdjson {
|
|||||||
namespace SIMDJSON_IMPLEMENTATION {
|
namespace SIMDJSON_IMPLEMENTATION {
|
||||||
namespace {
|
namespace {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* We could use a scalar version of parse_eight_digits_unrolled that compiles
|
||||||
|
* to something like
|
||||||
|
*
|
||||||
|
* and rax, qword ptr [rdi]
|
||||||
|
* imul rax, rax, 2561
|
||||||
|
* shr rax, 8
|
||||||
|
* movabs rcx, 71777214294589695
|
||||||
|
* and rcx, rax
|
||||||
|
* imul rax, rcx, 6553601
|
||||||
|
* shr rax, 16
|
||||||
|
* movabs rcx, 281470681808895
|
||||||
|
* and rcx, rax
|
||||||
|
* movabs rax, 42949672960001
|
||||||
|
* imul rax, rcx
|
||||||
|
* shr rax, 32
|
||||||
|
*
|
||||||
|
* The vectorized sequence below is favorable, as it compiles to
|
||||||
|
* vmovq xmm0, qword ptr [rdi] # xmm0 = mem[0],zero
|
||||||
|
* vpaddb xmm0, xmm0, xmmword ptr [rip + .LCPI0_0]
|
||||||
|
* vpmaddubsw xmm0, xmm0, xmmword ptr [rip + .LCPI0_1]
|
||||||
|
* vpmaddwd xmm0, xmm0, xmmword ptr [rip + .LCPI0_2]
|
||||||
|
* vpackusdw xmm0, xmm0, xmm0
|
||||||
|
* vpmaddwd xmm0, xmm0, xmmword ptr [rip + .LCPI0_3]
|
||||||
|
* vmovd eax, xmm0
|
||||||
|
* even though it does twice the work and looks far more complicated.
|
||||||
|
*/
|
||||||
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||||
// this actually computes *16* values so we are being wasteful.
|
// this actually computes *16* values so we are being wasteful.
|
||||||
const __m128i ascii0 = _mm_set1_epi8('0');
|
const __m128i ascii0 = _mm_set1_epi8('0');
|
||||||
@@ -13,8 +40,10 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
|
|||||||
const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
|
const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
|
||||||
const __m128i mul_1_10000 =
|
const __m128i mul_1_10000 =
|
||||||
_mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
|
_mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
|
||||||
|
// _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)) is faster
|
||||||
|
// but _mm_loadl_epi64(chars) avoids buffer overflows.
|
||||||
const __m128i input = _mm_sub_epi8(
|
const __m128i input = _mm_sub_epi8(
|
||||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
|
_mm_loadl_epi64(reinterpret_cast<const __m128i *>(chars)), ascii0);
|
||||||
const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
|
const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
|
||||||
const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
|
const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
|
||||||
const __m128i t3 = _mm_packus_epi32(t2, t2);
|
const __m128i t3 = _mm_packus_epi32(t2, t2);
|
||||||
@@ -27,7 +56,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
|
|||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
} // namespace simdjson
|
} // namespace simdjson
|
||||||
|
|
||||||
#define SWAR_NUMBER_PARSING
|
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||||
|
|
||||||
#include "simdjson/generic/numberparsing.h"
|
#include "simdjson/generic/numberparsing.h"
|
||||||
|
|
||||||
|
|||||||
@@ -15,7 +15,10 @@ using namespace simd;
|
|||||||
struct backslash_and_quote {
|
struct backslash_and_quote {
|
||||||
public:
|
public:
|
||||||
static constexpr uint32_t BYTES_PROCESSED = 32;
|
static constexpr uint32_t BYTES_PROCESSED = 32;
|
||||||
simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
|
||||||
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst);
|
||||||
|
|
||||||
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
||||||
simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; }
|
simdjson_really_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; }
|
||||||
@@ -26,7 +29,21 @@ public:
|
|||||||
uint32_t quote_bits;
|
uint32_t quote_bits;
|
||||||
}; // struct backslash_and_quote
|
}; // struct backslash_and_quote
|
||||||
|
|
||||||
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
simdjson_really_inline backslash_and_quote
|
||||||
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
|
||||||
|
// If we don't have enough memory left to load a whole simd register, copy it first.
|
||||||
|
uint8_t tmpbuf[BYTES_PROCESSED];
|
||||||
|
if (simdjson_unlikely(src > last_full_buf)) {
|
||||||
|
SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
|
||||||
|
std::memset(tmpbuf, 0, BYTES_PROCESSED);
|
||||||
|
std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
|
||||||
|
src = tmpbuf;
|
||||||
|
}
|
||||||
|
return copy_and_find(src, dst);
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline backslash_and_quote
|
||||||
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
||||||
// this can read up to 15 bytes beyond the buffer size, but we require
|
// this can read up to 15 bytes beyond the buffer size, but we require
|
||||||
// SIMDJSON_PADDING of padding
|
// SIMDJSON_PADDING of padding
|
||||||
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
|
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
|
||||||
|
|||||||
@@ -34,6 +34,13 @@ inline bool is_streaming(stage1_mode mode) {
|
|||||||
|
|
||||||
namespace internal {
|
namespace internal {
|
||||||
|
|
||||||
|
// expectation: sizeof(open_container) = 64/8.
|
||||||
|
struct open_container {
|
||||||
|
uint32_t tape_index; // where, on the tape, does the scope ([,{) begins
|
||||||
|
uint32_t count; // how many elements in the scope
|
||||||
|
}; // struct open_container
|
||||||
|
|
||||||
|
static_assert(sizeof(open_container) == 64/8, "Open container must be 64 bits");
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An implementation of simdjson's DOM parser for a particular CPU architecture.
|
* An implementation of simdjson's DOM parser for a particular CPU architecture.
|
||||||
@@ -178,6 +185,19 @@ protected:
|
|||||||
*/
|
*/
|
||||||
size_t _max_depth{0};
|
size_t _max_depth{0};
|
||||||
|
|
||||||
|
public:
|
||||||
|
/** Tape location of each open { or [ */
|
||||||
|
std::unique_ptr<open_container[]> open_containers{};
|
||||||
|
/** Whether each open container is a [ or { */
|
||||||
|
std::unique_ptr<bool[]> is_array{};
|
||||||
|
/** Buffer passed to stage 1 */
|
||||||
|
const uint8_t *buf{};
|
||||||
|
/** Length passed to stage 1 */
|
||||||
|
size_t len{0};
|
||||||
|
/** Document passed to stage 2 */
|
||||||
|
dom::document *doc{};
|
||||||
|
|
||||||
|
protected:
|
||||||
// Declaring these so that subclasses can use them to implement their constructors.
|
// Declaring these so that subclasses can use them to implement their constructors.
|
||||||
simdjson_really_inline dom_parser_implementation() noexcept;
|
simdjson_really_inline dom_parser_implementation() noexcept;
|
||||||
simdjson_really_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept;
|
simdjson_really_inline dom_parser_implementation(dom_parser_implementation &&other) noexcept;
|
||||||
|
|||||||
@@ -110,6 +110,7 @@ inline const char *padded_string::data() const noexcept { return data_ptr; }
|
|||||||
inline char *padded_string::data() noexcept { return data_ptr; }
|
inline char *padded_string::data() noexcept { return data_ptr; }
|
||||||
|
|
||||||
inline padded_string::operator std::string_view() const { return std::string_view(data(), length()); }
|
inline padded_string::operator std::string_view() const { return std::string_view(data(), length()); }
|
||||||
|
inline std::string padded_string::to_string() const { return std::string(data(), length()); }
|
||||||
|
|
||||||
inline padded_string::operator padded_string_view() const noexcept {
|
inline padded_string::operator padded_string_view() const noexcept {
|
||||||
return padded_string_view(data(), length(), length() + SIMDJSON_PADDING);
|
return padded_string_view(data(), length(), length() + SIMDJSON_PADDING);
|
||||||
|
|||||||
@@ -98,6 +98,11 @@ struct padded_string final {
|
|||||||
*/
|
*/
|
||||||
operator std::string_view() const;
|
operator std::string_view() const;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a std::string with the same content (and no padding).
|
||||||
|
*/
|
||||||
|
std::string to_string() const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a padded_string_view with the same content.
|
* Create a padded_string_view with the same content.
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -50,6 +50,8 @@ public:
|
|||||||
* @param capacity The allocated length of the string, including padding.
|
* @param capacity The allocated length of the string, including padding.
|
||||||
*/
|
*/
|
||||||
explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept;
|
explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept;
|
||||||
|
inline padded_string_view(const char *s, size_t capacity) noexcept = delete;
|
||||||
|
inline padded_string_view(const uint8_t *s, size_t capacity) noexcept = delete;
|
||||||
|
|
||||||
/** The number of allocated bytes. */
|
/** The number of allocated bytes. */
|
||||||
inline size_t capacity() const noexcept;
|
inline size_t capacity() const noexcept;
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) {
|
|||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
} // namespace simdjson
|
} // namespace simdjson
|
||||||
|
|
||||||
#define SWAR_NUMBER_PARSING
|
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||||
|
|
||||||
#include "simdjson/generic/numberparsing.h"
|
#include "simdjson/generic/numberparsing.h"
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,8 @@ struct backslash_and_quote {
|
|||||||
public:
|
public:
|
||||||
static constexpr uint32_t BYTES_PROCESSED = 32;
|
static constexpr uint32_t BYTES_PROCESSED = 32;
|
||||||
simdjson_really_inline static backslash_and_quote
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
|
||||||
|
simdjson_really_inline static backslash_and_quote
|
||||||
copy_and_find(const uint8_t *src, uint8_t *dst);
|
copy_and_find(const uint8_t *src, uint8_t *dst);
|
||||||
|
|
||||||
simdjson_really_inline bool has_quote_first() {
|
simdjson_really_inline bool has_quote_first() {
|
||||||
@@ -33,6 +35,19 @@ public:
|
|||||||
uint32_t quote_bits;
|
uint32_t quote_bits;
|
||||||
}; // struct backslash_and_quote
|
}; // struct backslash_and_quote
|
||||||
|
|
||||||
|
simdjson_really_inline backslash_and_quote
|
||||||
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
|
||||||
|
// If we don't have enough memory left to load a whole simd register, copy it first.
|
||||||
|
uint8_t tmpbuf[BYTES_PROCESSED];
|
||||||
|
if (simdjson_unlikely(src > last_full_buf)) {
|
||||||
|
SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
|
||||||
|
std::memset(tmpbuf, 0, BYTES_PROCESSED);
|
||||||
|
std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
|
||||||
|
src = tmpbuf;
|
||||||
|
}
|
||||||
|
return copy_and_find(src, dst);
|
||||||
|
}
|
||||||
|
|
||||||
simdjson_really_inline backslash_and_quote
|
simdjson_really_inline backslash_and_quote
|
||||||
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
||||||
// this can read up to 31 bytes beyond the buffer size, but we require
|
// this can read up to 31 bytes beyond the buffer size, but we require
|
||||||
|
|||||||
@@ -4,7 +4,33 @@
|
|||||||
namespace simdjson {
|
namespace simdjson {
|
||||||
namespace SIMDJSON_IMPLEMENTATION {
|
namespace SIMDJSON_IMPLEMENTATION {
|
||||||
namespace {
|
namespace {
|
||||||
|
/**
|
||||||
|
* We could use a scalar version of parse_eight_digits_unrolled that compiles
|
||||||
|
* to something like
|
||||||
|
*
|
||||||
|
* and rax, qword ptr [rdi]
|
||||||
|
* imul rax, rax, 2561
|
||||||
|
* shr rax, 8
|
||||||
|
* movabs rcx, 71777214294589695
|
||||||
|
* and rcx, rax
|
||||||
|
* imul rax, rcx, 6553601
|
||||||
|
* shr rax, 16
|
||||||
|
* movabs rcx, 281470681808895
|
||||||
|
* and rcx, rax
|
||||||
|
* movabs rax, 42949672960001
|
||||||
|
* imul rax, rcx
|
||||||
|
* shr rax, 32
|
||||||
|
*
|
||||||
|
* The vectorized sequence below is favorable, as it compiles to
|
||||||
|
* vmovq xmm0, qword ptr [rdi] # xmm0 = mem[0],zero
|
||||||
|
* vpaddb xmm0, xmm0, xmmword ptr [rip + .LCPI0_0]
|
||||||
|
* vpmaddubsw xmm0, xmm0, xmmword ptr [rip + .LCPI0_1]
|
||||||
|
* vpmaddwd xmm0, xmm0, xmmword ptr [rip + .LCPI0_2]
|
||||||
|
* vpackusdw xmm0, xmm0, xmm0
|
||||||
|
* vpmaddwd xmm0, xmm0, xmmword ptr [rip + .LCPI0_3]
|
||||||
|
* vmovd eax, xmm0
|
||||||
|
* even though it does twice the work and looks far more complicated.
|
||||||
|
*/
|
||||||
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t *chars) {
|
||||||
// this actually computes *16* values so we are being wasteful.
|
// this actually computes *16* values so we are being wasteful.
|
||||||
const __m128i ascii0 = _mm_set1_epi8('0');
|
const __m128i ascii0 = _mm_set1_epi8('0');
|
||||||
@@ -13,8 +39,10 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
|
|||||||
const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
|
const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1);
|
||||||
const __m128i mul_1_10000 =
|
const __m128i mul_1_10000 =
|
||||||
_mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
|
_mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1);
|
||||||
|
// _mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)) is faster
|
||||||
|
// but _mm_loadl_epi64(chars) avoids buffer overflows.
|
||||||
const __m128i input = _mm_sub_epi8(
|
const __m128i input = _mm_sub_epi8(
|
||||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(chars)), ascii0);
|
_mm_loadl_epi64(reinterpret_cast<const __m128i *>(chars)), ascii0);
|
||||||
const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
|
const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10);
|
||||||
const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
|
const __m128i t2 = _mm_madd_epi16(t1, mul_1_100);
|
||||||
const __m128i t3 = _mm_packus_epi32(t2, t2);
|
const __m128i t3 = _mm_packus_epi32(t2, t2);
|
||||||
@@ -23,11 +51,12 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t
|
|||||||
t4); // only captures the sum of the first 8 digits, drop the rest
|
t4); // only captures the sum of the first 8 digits, drop the rest
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
} // unnamed namespace
|
} // unnamed namespace
|
||||||
} // namespace SIMDJSON_IMPLEMENTATION
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
} // namespace simdjson
|
} // namespace simdjson
|
||||||
|
|
||||||
#define SWAR_NUMBER_PARSING
|
#define SIMDJSON_SWAR_NUMBER_PARSING 1
|
||||||
|
|
||||||
#include "simdjson/generic/numberparsing.h"
|
#include "simdjson/generic/numberparsing.h"
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,10 @@ using namespace simd;
|
|||||||
struct backslash_and_quote {
|
struct backslash_and_quote {
|
||||||
public:
|
public:
|
||||||
static constexpr uint32_t BYTES_PROCESSED = 32;
|
static constexpr uint32_t BYTES_PROCESSED = 32;
|
||||||
simdjson_really_inline static backslash_and_quote copy_and_find(const uint8_t *src, uint8_t *dst);
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf);
|
||||||
|
simdjson_really_inline static backslash_and_quote
|
||||||
|
copy_and_find(const uint8_t *src, uint8_t *dst);
|
||||||
|
|
||||||
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
simdjson_really_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; }
|
||||||
simdjson_really_inline bool has_backslash() { return bs_bits != 0; }
|
simdjson_really_inline bool has_backslash() { return bs_bits != 0; }
|
||||||
@@ -22,7 +25,21 @@ public:
|
|||||||
uint32_t quote_bits;
|
uint32_t quote_bits;
|
||||||
}; // struct backslash_and_quote
|
}; // struct backslash_and_quote
|
||||||
|
|
||||||
simdjson_really_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
simdjson_really_inline backslash_and_quote
|
||||||
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst, const uint8_t *last_full_buf) {
|
||||||
|
// If we don't have enough memory left to load a whole simd register, copy it first.
|
||||||
|
uint8_t tmpbuf[BYTES_PROCESSED];
|
||||||
|
if (simdjson_unlikely(src > last_full_buf)) {
|
||||||
|
SIMDJSON_ASSUME(last_full_buf+BYTES_PROCESSED > src);
|
||||||
|
std::memset(tmpbuf, 0, BYTES_PROCESSED);
|
||||||
|
std::memcpy(tmpbuf, src, last_full_buf+BYTES_PROCESSED-src);
|
||||||
|
src = tmpbuf;
|
||||||
|
}
|
||||||
|
return copy_and_find(src, dst);
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_really_inline backslash_and_quote
|
||||||
|
backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) {
|
||||||
// this can read up to 31 bytes beyond the buffer size, but we require
|
// this can read up to 31 bytes beyond the buffer size, but we require
|
||||||
// SIMDJSON_PADDING of padding
|
// SIMDJSON_PADDING of padding
|
||||||
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
|
static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes");
|
||||||
|
|||||||
+887
-284
File diff suppressed because it is too large
Load Diff
+3738
-1327
File diff suppressed because it is too large
Load Diff
@@ -241,9 +241,11 @@ simdjson_really_inline json_iterator::json_iterator(dom_parser_implementation &_
|
|||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept {
|
simdjson_really_inline const uint8_t *json_iterator::peek() const noexcept {
|
||||||
|
if(at_eof()) { return reinterpret_cast<const uint8_t *>(" ");}
|
||||||
return &buf[*(next_structural)];
|
return &buf[*(next_structural)];
|
||||||
}
|
}
|
||||||
simdjson_really_inline const uint8_t *json_iterator::advance() noexcept {
|
simdjson_really_inline const uint8_t *json_iterator::advance() noexcept {
|
||||||
|
if(at_eof()) { return reinterpret_cast<const uint8_t *>(" ");}
|
||||||
return &buf[*(next_structural++)];
|
return &buf[*(next_structural++)];
|
||||||
}
|
}
|
||||||
simdjson_really_inline size_t json_iterator::remaining_len() const noexcept {
|
simdjson_really_inline size_t json_iterator::remaining_len() const noexcept {
|
||||||
|
|||||||
@@ -28,19 +28,22 @@ public:
|
|||||||
return buf[*next_structural];
|
return buf[*next_structural];
|
||||||
}
|
}
|
||||||
simdjson_really_inline const uint8_t* peek() {
|
simdjson_really_inline const uint8_t* peek() {
|
||||||
|
if(at_end_of_input_buffer()) { return " ";}
|
||||||
return &buf[*next_structural];
|
return &buf[*next_structural];
|
||||||
}
|
}
|
||||||
simdjson_really_inline const uint8_t* advance() {
|
simdjson_really_inline const uint8_t* advance() {
|
||||||
|
if(at_end_of_input_buffer()) { return " ";}
|
||||||
return &buf[*(next_structural++)];
|
return &buf[*(next_structural++)];
|
||||||
}
|
}
|
||||||
simdjson_really_inline char advance_char() {
|
simdjson_really_inline char advance_char() {
|
||||||
|
if(at_end_of_input_buffer()) { return " ";}
|
||||||
return buf[*(next_structural++)];
|
return buf[*(next_structural++)];
|
||||||
}
|
}
|
||||||
simdjson_really_inline size_t remaining_len() {
|
simdjson_really_inline size_t remaining_len() {
|
||||||
return dom_parser.len - *(next_structural-1);
|
return dom_parser.len - *(next_structural-1);
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_really_inline bool at_end() {
|
simdjson_really_inline bool at_end_of_input_buffer() {
|
||||||
return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes];
|
return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes];
|
||||||
}
|
}
|
||||||
simdjson_really_inline bool at_beginning() {
|
simdjson_really_inline bool at_beginning() {
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ simdjson_really_inline tape_builder::tape_builder(dom::document &doc) noexcept :
|
|||||||
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept {
|
||||||
iter.log_value(key ? "key" : "string");
|
iter.log_value(key ? "key" : "string");
|
||||||
uint8_t *dst = on_start_string(iter);
|
uint8_t *dst = on_start_string(iter);
|
||||||
dst = stringparsing::parse_string(value+1, dst);
|
dst = stringparsing::parse_string(value+1, dst, iter.buf+iter.dom_parser.len);
|
||||||
if (dst == nullptr) {
|
if (dst == nullptr) {
|
||||||
iter.log_error("Invalid escape in string");
|
iter.log_error("Invalid escape in string");
|
||||||
return STRING_ERROR;
|
return STRING_ERROR;
|
||||||
@@ -160,34 +160,23 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
|
|||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept {
|
||||||
iter.log_value("number");
|
iter.log_value("number");
|
||||||
return numberparsing::parse_number(value, tape);
|
#ifdef SIMDJSON_SWAR_NUMBER_PARSING
|
||||||
|
return numberparsing::parse_number(value, tape, value + iter.remaining_len());
|
||||||
|
#else
|
||||||
|
return numberparsing::parse_number(value, tape, nullptr);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept {
|
||||||
//
|
// Root numbers will not have another token after them, so we need to be careful not to step
|
||||||
// We need to make a copy to make sure that the string is space terminated.
|
// off the edge of the buffer.
|
||||||
// This is not about padding the input, which should already padded up
|
iter.log_value("number");
|
||||||
// to len + SIMDJSON_PADDING. However, we have no control at this stage
|
return numberparsing::parse_number(value, tape, value + iter.remaining_len());
|
||||||
// on how the padding was done. What if the input string was padded with nulls?
|
|
||||||
// It is quite common for an input string to have an extra null character (C string).
|
|
||||||
// We do not want to allow 9\0 (where \0 is the null character) inside a JSON
|
|
||||||
// document, but the string "9\0" by itself is fine. So we make a copy and
|
|
||||||
// pad the input with spaces when we know that there is just one input element.
|
|
||||||
// This copy is relatively expensive, but it will almost never be called in
|
|
||||||
// practice unless you are in the strange scenario where you have many JSON
|
|
||||||
// documents made of single atoms.
|
|
||||||
//
|
|
||||||
std::unique_ptr<uint8_t[]>copy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]);
|
|
||||||
if (copy.get() == nullptr) { return MEMALLOC; }
|
|
||||||
std::memcpy(copy.get(), value, iter.remaining_len());
|
|
||||||
std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING);
|
|
||||||
error_code error = visit_number(iter, copy.get());
|
|
||||||
return error;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept {
|
||||||
iter.log_value("true");
|
iter.log_value("true");
|
||||||
if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; }
|
if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; }
|
||||||
tape.append(0, internal::tape_type::TRUE_VALUE);
|
tape.append(0, internal::tape_type::TRUE_VALUE);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
@@ -201,7 +190,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
|
|||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept {
|
||||||
iter.log_value("false");
|
iter.log_value("false");
|
||||||
if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; }
|
if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; }
|
||||||
tape.append(0, internal::tape_type::FALSE_VALUE);
|
tape.append(0, internal::tape_type::FALSE_VALUE);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
@@ -215,7 +204,7 @@ simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_root_
|
|||||||
|
|
||||||
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
|
simdjson_warn_unused simdjson_really_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept {
|
||||||
iter.log_value("null");
|
iter.log_value("null");
|
||||||
if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; }
|
if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; }
|
||||||
tape.append(0, internal::tape_type::NULL_VALUE);
|
tape.append(0, internal::tape_type::NULL_VALUE);
|
||||||
return SUCCESS;
|
return SUCCESS;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,36 +2,43 @@
|
|||||||
|
|
||||||
namespace simdjson {
|
namespace simdjson {
|
||||||
namespace internal {
|
namespace internal {
|
||||||
|
/**
|
||||||
|
* We include in the error messages the name of the enum
|
||||||
|
* as a quality-of-life feature for programmers who receive
|
||||||
|
* the error. If they get a text error, they sometimes want
|
||||||
|
* to trace back the error code, and it is handy if it is
|
||||||
|
* right there.
|
||||||
|
*/
|
||||||
|
|
||||||
SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[] {
|
SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[] {
|
||||||
{ SUCCESS, "No error" },
|
{ SUCCESS, "No error (SUCCESS)" },
|
||||||
{ CAPACITY, "This parser can't support a document that big" },
|
{ CAPACITY, "This parser can't support a document that big (CAPACITY)" },
|
||||||
{ MEMALLOC, "Error allocating memory, we're most likely out of memory" },
|
{ MEMALLOC, "Error allocating memory, we're most likely out of memory (MEMALLOC)" },
|
||||||
{ TAPE_ERROR, "The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc." },
|
{ TAPE_ERROR, "The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc. (TAPE_ERROR)" },
|
||||||
{ DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays)" },
|
{ DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays) (DEPTH_ERROR)" },
|
||||||
{ STRING_ERROR, "Problem while parsing a string" },
|
{ STRING_ERROR, "Problem while parsing a string (STRING_ERROR)" },
|
||||||
{ T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'" },
|
{ T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't' (T_ATOM_ERROR)" },
|
||||||
{ F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'" },
|
{ F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f' (F_ATOM_ERROR)" },
|
||||||
{ N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'" },
|
{ N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n' (N_ATOM_ERROR)" },
|
||||||
{ NUMBER_ERROR, "Problem while parsing a number" },
|
{ NUMBER_ERROR, "Problem while parsing a number (NUMBER_ERROR)" },
|
||||||
{ UTF8_ERROR, "The input is not valid UTF-8" },
|
{ UTF8_ERROR, "The input is not valid UTF-8 (UTF8_ERROR)" },
|
||||||
{ UNINITIALIZED, "Uninitialized" },
|
{ UNINITIALIZED, "Uninitialized (UNINITIALIZED)" },
|
||||||
{ EMPTY, "Empty: no JSON found" },
|
{ EMPTY, "Empty: no JSON found (EMPTY)" },
|
||||||
{ UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters" },
|
{ UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters (UNESCAPED_CHARS)" },
|
||||||
{ UNCLOSED_STRING, "A string is opened, but never closed." },
|
{ UNCLOSED_STRING, "A string is opened, but never closed (UNCLOSED_STRING)" },
|
||||||
{ UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (perhaps it's a non-SIMD CPU?)." },
|
{ UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (UNSUPPORTED_ARCHITECTURE)" },
|
||||||
{ INCORRECT_TYPE, "The JSON element does not have the requested type." },
|
{ INCORRECT_TYPE, "The JSON element does not have the requested type (INCORRECT_TYPE)" },
|
||||||
{ NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type." },
|
{ NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type (NUMBER_OUT_OF_RANGE)" },
|
||||||
{ INDEX_OUT_OF_BOUNDS, "Attempted to access an element of a JSON array that is beyond its length." },
|
{ INDEX_OUT_OF_BOUNDS, "Attempted to access an element of a JSON array that is beyond its length (INDEX_OUT_OF_BOUNDS)" },
|
||||||
{ NO_SUCH_FIELD, "The JSON field referenced does not exist in this object." },
|
{ NO_SUCH_FIELD, "The JSON field referenced does not exist in this object (NO_SUCH_FIELD)" },
|
||||||
{ IO_ERROR, "Error reading the file." },
|
{ IO_ERROR, "Error reading the file (IO_ERROR)" },
|
||||||
{ INVALID_JSON_POINTER, "Invalid JSON pointer syntax." },
|
{ INVALID_JSON_POINTER, "Invalid JSON pointer syntax (INVALID_JSON_POINTER)" },
|
||||||
{ INVALID_URI_FRAGMENT, "Invalid URI fragment syntax." },
|
{ INVALID_URI_FRAGMENT, "Invalid URI fragment syntax (INVALID_URI_FRAGMENT)" },
|
||||||
{ UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson" },
|
{ UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson (UNEXPECTED_ERROR)" },
|
||||||
{ PARSER_IN_USE, "Cannot parse a new document while a document is still in use." },
|
{ PARSER_IN_USE, "Cannot parse a new document while a document is still in use (PARSER_IN_USE)" },
|
||||||
{ OUT_OF_ORDER_ITERATION, "Objects and arrays can only be iterated when they are first encountered." },
|
{ OUT_OF_ORDER_ITERATION, "Objects and arrays can only be iterated when they are first encountered (OUT_OF_ORDER_ITERATION)" },
|
||||||
{ INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length." },
|
{ INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length (INSUFFICIENT_PADDING)" },
|
||||||
{ INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array." }
|
{ INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array (INCOMPLETE_ARRAY_OR_OBJECT)" }
|
||||||
}; // error_messages[]
|
}; // error_messages[]
|
||||||
|
|
||||||
} // namespace internal
|
} // namespace internal
|
||||||
|
|||||||
+91
-38
@@ -51,14 +51,14 @@ namespace number_tests {
|
|||||||
};
|
};
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
for(auto string_double : ground_truth) {
|
for(auto string_double : ground_truth) {
|
||||||
std::cout << "parsing the string '" << string_double.first << "'" << std::endl;
|
|
||||||
std::cout << "I am expecting the floating-point value '" << string_double.second << "'" << std::endl;
|
|
||||||
double result;
|
double result;
|
||||||
ASSERT_SUCCESS(parser.parse(string_double.first).get(result));
|
std::string input = string_double.first;
|
||||||
std::cout << "Resulting float is '" << result << "'" << std::endl;
|
ASSERT_SUCCESS(parser.parse(input).get(result));
|
||||||
if(result != string_double.second) {
|
if(result != string_double.second) {
|
||||||
|
std::cout << "parsing the string '" << input << "'" << std::endl;
|
||||||
|
std::cout << "I am expecting the floating-point value '" << string_double.second << "'" << std::endl;
|
||||||
|
std::cout << "Resulting float is '" << result << "'" << std::endl;
|
||||||
std::cerr << std::hexfloat << result << " vs " << string_double.second << std::endl;
|
std::cerr << std::hexfloat << result << " vs " << string_double.second << std::endl;
|
||||||
std::cerr << string_double.first << std::endl;
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -113,9 +113,10 @@ namespace number_tests {
|
|||||||
};
|
};
|
||||||
for (std::pair<std::string, double> p : testing) {
|
for (std::pair<std::string, double> p : testing) {
|
||||||
double actual;
|
double actual;
|
||||||
ASSERT_SUCCESS(parser.parse(p.first).get(actual));
|
std::string input = p.first;
|
||||||
|
ASSERT_SUCCESS(parser.parse(input).get(actual));
|
||||||
if (actual != p.second) {
|
if (actual != p.second) {
|
||||||
std::cerr << "JSON '" << p.first << "' parsed to " << actual << " instead of " << p.first << std::endl;
|
std::cerr << "JSON '" << input << "' parsed to " << actual << " instead of " << p.second << std::endl;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -326,9 +327,9 @@ namespace parse_api_tests {
|
|||||||
using namespace simdjson;
|
using namespace simdjson;
|
||||||
using namespace simdjson::dom;
|
using namespace simdjson::dom;
|
||||||
|
|
||||||
const padded_string BASIC_JSON = "[1,2,3]"_padded;
|
const std::string BASIC_JSON = "[1,2,3]";
|
||||||
const padded_string BASIC_NDJSON = "[1,2,3]\n[4,5,6]"_padded;
|
const std::string BASIC_NDJSON = "[1,2,3]\n[4,5,6]";
|
||||||
const padded_string EMPTY_NDJSON = ""_padded;
|
const std::string EMPTY_NDJSON = "";
|
||||||
bool parser_moving_parser() {
|
bool parser_moving_parser() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
typedef std::tuple<std::string, std::unique_ptr<parser>,element> simdjson_tuple;
|
typedef std::tuple<std::string, std::unique_ptr<parser>,element> simdjson_tuple;
|
||||||
@@ -349,7 +350,7 @@ namespace parse_api_tests {
|
|||||||
#if SIMDJSON_EXCEPTIONS
|
#if SIMDJSON_EXCEPTIONS
|
||||||
bool issue679() {
|
bool issue679() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto input = "[1, 2, 3]"_padded;
|
std::string input = "[1, 2, 3]";
|
||||||
dom::document doc;
|
dom::document doc;
|
||||||
{
|
{
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
@@ -377,7 +378,7 @@ namespace parse_api_tests {
|
|||||||
//See https://github.com/simdjson/simdjson/issues/1332
|
//See https://github.com/simdjson/simdjson/issues/1332
|
||||||
bool parser_moving_parser_and_recovering() {
|
bool parser_moving_parser_and_recovering() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto input = "[1, 2, 3]"_padded;
|
std::string input = "[1, 2, 3]";
|
||||||
auto parser = dom::parser{};
|
auto parser = dom::parser{};
|
||||||
dom::element root = parser.parse(input); // might throw
|
dom::element root = parser.parse(input); // might throw
|
||||||
auto parser2 = std::move(parser);
|
auto parser2 = std::move(parser);
|
||||||
@@ -390,7 +391,7 @@ namespace parse_api_tests {
|
|||||||
struct moving_parser {
|
struct moving_parser {
|
||||||
dom::parser parser{};
|
dom::parser parser{};
|
||||||
bool is_valid{false};
|
bool is_valid{false};
|
||||||
simdjson::error_code parse(const padded_string & input) {
|
simdjson::error_code parse(const std::string & input) {
|
||||||
auto answer = parser.parse(input).error();
|
auto answer = parser.parse(input).error();
|
||||||
is_valid = !answer;
|
is_valid = !answer;
|
||||||
return answer;
|
return answer;
|
||||||
@@ -404,7 +405,7 @@ namespace parse_api_tests {
|
|||||||
// Shows how to use moving_parser
|
// Shows how to use moving_parser
|
||||||
bool parser_moving_parser_and_recovering_struct() {
|
bool parser_moving_parser_and_recovering_struct() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto input = "[1, 2, 3]"_padded;
|
std::string input = "[1, 2, 3]";
|
||||||
moving_parser mp{};
|
moving_parser mp{};
|
||||||
mp.parse(input);// I could check the error here if I want
|
mp.parse(input);// I could check the error here if I want
|
||||||
auto mp2 = std::move(mp);
|
auto mp2 = std::move(mp);
|
||||||
@@ -645,7 +646,7 @@ namespace dom_api_tests {
|
|||||||
// returns true if successful
|
// returns true if successful
|
||||||
bool ParsedJson_Iterator_test() {
|
bool ParsedJson_Iterator_test() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
simdjson::padded_string json = R"({
|
std::string json = R"({
|
||||||
"Image": {
|
"Image": {
|
||||||
"Width": 800,
|
"Width": 800,
|
||||||
"Height": 600,
|
"Height": 600,
|
||||||
@@ -658,7 +659,7 @@ namespace dom_api_tests {
|
|||||||
"Animated" : false,
|
"Animated" : false,
|
||||||
"IDs": [116, 943, 234, 38793]
|
"IDs": [116, 943, 234, 38793]
|
||||||
}
|
}
|
||||||
})"_padded;
|
})";
|
||||||
simdjson::ParsedJson pj = build_parsed_json(json);
|
simdjson::ParsedJson pj = build_parsed_json(json);
|
||||||
if (pj.error) {
|
if (pj.error) {
|
||||||
printf("Could not parse '%s': %s\n", json.data(), simdjson::error_message(pj.error));
|
printf("Could not parse '%s': %s\n", json.data(), simdjson::error_message(pj.error));
|
||||||
@@ -1045,10 +1046,10 @@ namespace dom_api_tests {
|
|||||||
bool string_value_exception() {
|
bool string_value_exception() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
ASSERT_EQUAL( (const char *)parser.parse(R"("hi")"_padded), "hi" );
|
ASSERT_EQUAL( (const char *)parser.parse(std::string(R"("hi")")), "hi" );
|
||||||
ASSERT_EQUAL( string_view(parser.parse(R"("hi")"_padded)), "hi" );
|
ASSERT_EQUAL( string_view(parser.parse(std::string(R"("hi")"))), "hi" );
|
||||||
ASSERT_EQUAL( (const char *)parser.parse(R"("has backslash\\")"_padded), "has backslash\\");
|
ASSERT_EQUAL( (const char *)parser.parse(std::string(R"("has backslash\\")")), "has backslash\\");
|
||||||
ASSERT_EQUAL( string_view(parser.parse(R"("has backslash\\")"_padded)), "has backslash\\" );
|
ASSERT_EQUAL( string_view(parser.parse(std::string(R"("has backslash\\")"))), "has backslash\\" );
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1056,18 +1057,18 @@ namespace dom_api_tests {
|
|||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
ASSERT_EQUAL( uint64_t(parser.parse("0"_padded)), 0);
|
ASSERT_EQUAL( uint64_t(parser.parse(std::string("0"))), 0);
|
||||||
ASSERT_EQUAL( int64_t(parser.parse("0"_padded)), 0);
|
ASSERT_EQUAL( int64_t(parser.parse(std::string("0"))), 0);
|
||||||
ASSERT_EQUAL( double(parser.parse("0"_padded)), 0);
|
ASSERT_EQUAL( double(parser.parse(std::string("0"))), 0);
|
||||||
|
|
||||||
ASSERT_EQUAL( uint64_t(parser.parse("1"_padded)), 1);
|
ASSERT_EQUAL( uint64_t(parser.parse(std::string("1"))), 1);
|
||||||
ASSERT_EQUAL( int64_t(parser.parse("1"_padded)), 1);
|
ASSERT_EQUAL( int64_t(parser.parse(std::string("1"))), 1);
|
||||||
ASSERT_EQUAL( double(parser.parse("1"_padded)), 1);
|
ASSERT_EQUAL( double(parser.parse(std::string("1"))), 1);
|
||||||
|
|
||||||
ASSERT_EQUAL( int64_t(parser.parse("-1"_padded)), -1);
|
ASSERT_EQUAL( int64_t(parser.parse(std::string("-1"))), -1);
|
||||||
ASSERT_EQUAL( double(parser.parse("-1"_padded)), -1);
|
ASSERT_EQUAL( double(parser.parse(std::string("-1"))), -1);
|
||||||
|
|
||||||
ASSERT_EQUAL( double(parser.parse("1.1"_padded)), 1.1);
|
ASSERT_EQUAL( double(parser.parse(std::string("1.1"))), 1.1);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -1076,9 +1077,9 @@ namespace dom_api_tests {
|
|||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
ASSERT_EQUAL( bool(parser.parse("true"_padded)), true);
|
ASSERT_EQUAL( bool(parser.parse(std::string("true"))), true);
|
||||||
|
|
||||||
ASSERT_EQUAL( bool(parser.parse("false"_padded)), false);
|
ASSERT_EQUAL( bool(parser.parse(std::string("false"))), false);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -1087,7 +1088,7 @@ namespace dom_api_tests {
|
|||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
ASSERT_EQUAL( bool(parser.parse("null"_padded).is_null()), true );
|
ASSERT_EQUAL( bool(parser.parse(std::string("null")).is_null()), true );
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@@ -1218,7 +1219,7 @@ namespace type_tests {
|
|||||||
using namespace simdjson;
|
using namespace simdjson;
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
const padded_string ALL_TYPES_JSON = R"(
|
const std::string ALL_TYPES_JSON = R"(
|
||||||
{
|
{
|
||||||
"array": [],
|
"array": [],
|
||||||
|
|
||||||
@@ -1245,7 +1246,7 @@ namespace type_tests {
|
|||||||
|
|
||||||
"null": null
|
"null": null
|
||||||
}
|
}
|
||||||
)"_padded;
|
)";
|
||||||
|
|
||||||
template<typename T>
|
template<typename T>
|
||||||
bool test_cast(simdjson_result<dom::element> result, T expected) {
|
bool test_cast(simdjson_result<dom::element> result, T expected) {
|
||||||
@@ -1740,7 +1741,7 @@ namespace format_tests {
|
|||||||
using namespace simdjson;
|
using namespace simdjson;
|
||||||
using namespace simdjson::dom;
|
using namespace simdjson::dom;
|
||||||
using namespace std;
|
using namespace std;
|
||||||
const padded_string DOCUMENT = R"({ "foo" : 1, "bar" : [ 1, 2, 0.11111111111111113 ], "baz": { "a": 3.1415926535897936, "b": 2, "c": 3.141592653589794 } })"_padded;
|
const std::string DOCUMENT = R"({ "foo" : 1, "bar" : [ 1, 2, 0.11111111111111113 ], "baz": { "a": 3.1415926535897936, "b": 2, "c": 3.141592653589794 } })";
|
||||||
const string MINIFIED(R"({"foo":1,"bar":[1,2,0.11111111111111113],"baz":{"a":3.1415926535897936,"b":2,"c":3.141592653589794}})");
|
const string MINIFIED(R"({"foo":1,"bar":[1,2,0.11111111111111113],"baz":{"a":3.1415926535897936,"b":2,"c":3.141592653589794}})");
|
||||||
bool assert_minified(ostringstream &actual, const std::string &expected=MINIFIED) {
|
bool assert_minified(ostringstream &actual, const std::string &expected=MINIFIED) {
|
||||||
if (actual.str() != expected) {
|
if (actual.str() != expected) {
|
||||||
@@ -1965,7 +1966,7 @@ namespace to_string_tests {
|
|||||||
using namespace simdjson;
|
using namespace simdjson;
|
||||||
using namespace simdjson::dom;
|
using namespace simdjson::dom;
|
||||||
using namespace std;
|
using namespace std;
|
||||||
const padded_string DOCUMENT = R"({ "foo" : 1, "bar" : [ 1, 2, 0.11111111111111113 ], "baz": { "a": 3.1415926535897936, "b": 2, "c": 3.141592653589794 } })"_padded;
|
const std::string DOCUMENT = R"({ "foo" : 1, "bar" : [ 1, 2, 0.11111111111111113 ], "baz": { "a": 3.1415926535897936, "b": 2, "c": 3.141592653589794 } })";
|
||||||
const string MINIFIED(R"({"foo":1,"bar":[1,2,0.11111111111111113],"baz":{"a":3.1415926535897936,"b":2,"c":3.141592653589794}})");
|
const string MINIFIED(R"({"foo":1,"bar":[1,2,0.11111111111111113],"baz":{"a":3.1415926535897936,"b":2,"c":3.141592653589794}})");
|
||||||
bool assert_minified(ostringstream &actual, const std::string &expected=MINIFIED) {
|
bool assert_minified(ostringstream &actual, const std::string &expected=MINIFIED) {
|
||||||
if (actual.str() != expected) {
|
if (actual.str() != expected) {
|
||||||
@@ -1977,11 +1978,12 @@ namespace to_string_tests {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool print_to_string_large_int() {
|
bool print_to_string_large_int() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::element doc;
|
dom::element doc;
|
||||||
ASSERT_SUCCESS( parser.parse("-922337203685477580"_padded).get(doc) );
|
ASSERT_SUCCESS( parser.parse(std::string("-922337203685477580")).get(doc) );
|
||||||
ostringstream s;
|
ostringstream s;
|
||||||
s << to_string(doc);
|
s << to_string(doc);
|
||||||
if(s.str() != "-922337203685477580") {
|
if(s.str() != "-922337203685477580") {
|
||||||
@@ -2116,6 +2118,56 @@ namespace to_string_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool simple_overflows() {
|
||||||
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
|
simdjson::dom::parser parser;
|
||||||
|
simdjson::dom::element doc;
|
||||||
|
|
||||||
|
for (const char * val : {"[f]", "{\"a\":f}"}) {
|
||||||
|
char * tmp = new char[strlen(val)];
|
||||||
|
memcpy(tmp, val, strlen(val));
|
||||||
|
ASSERT_ERROR( parser.parse(tmp, strlen(val)).get(doc), simdjson::F_ATOM_ERROR);
|
||||||
|
delete[] tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const char * val : {"[t]", "{\"a\":t}"}) {
|
||||||
|
char * tmp = new char[strlen(val)];
|
||||||
|
memcpy(tmp, val, strlen(val));
|
||||||
|
ASSERT_ERROR( parser.parse(tmp, strlen(val)).get(doc), simdjson::T_ATOM_ERROR);
|
||||||
|
delete[] tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const char * val : {"[n]", "{\"a\":n}"}) {
|
||||||
|
char * tmp = new char[strlen(val)];
|
||||||
|
memcpy(tmp, val, strlen(val));
|
||||||
|
ASSERT_ERROR( parser.parse(tmp, strlen(val)).get(doc), simdjson::N_ATOM_ERROR);
|
||||||
|
delete[] tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const char * val : {"[-]", "{\"a\":-}"}) {
|
||||||
|
char * tmp = new char[strlen(val)];
|
||||||
|
memcpy(tmp, val, strlen(val));
|
||||||
|
ASSERT_ERROR( parser.parse(tmp, strlen(val)).get(doc), simdjson::NUMBER_ERROR);
|
||||||
|
delete[] tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const char * val : {"[[]", "{\"a\":[[]"}) {
|
||||||
|
char * tmp = new char[strlen(val)];
|
||||||
|
memcpy(tmp, val, strlen(val));
|
||||||
|
ASSERT_ERROR( parser.parse(tmp, strlen(val)).get(doc), simdjson::TAPE_ERROR);
|
||||||
|
delete[] tmp;
|
||||||
|
}
|
||||||
|
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("[f]")).get(doc), simdjson::F_ATOM_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("[t]")).get(doc), simdjson::T_ATOM_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("[n]")).get(doc), simdjson::N_ATOM_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("[-]")).get(doc), simdjson::NUMBER_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("{\"a\":f}")).get(doc), simdjson::F_ATOM_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("{\"a\":t}")).get(doc), simdjson::T_ATOM_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("{\"a\":n}")).get(doc), simdjson::N_ATOM_ERROR);
|
||||||
|
ASSERT_ERROR( parser.parse(std::string("{\"a\":-}")).get(doc), simdjson::NUMBER_ERROR);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
int main(int argc, char *argv[]) {
|
int main(int argc, char *argv[]) {
|
||||||
@@ -2153,7 +2205,8 @@ int main(int argc, char *argv[]) {
|
|||||||
std::cout << "------------------------------------------------------------" << std::endl;
|
std::cout << "------------------------------------------------------------" << std::endl;
|
||||||
|
|
||||||
std::cout << "Running basic tests." << std::endl;
|
std::cout << "Running basic tests." << std::endl;
|
||||||
if (to_string_tests::run() &&
|
if (simple_overflows() &&
|
||||||
|
to_string_tests::run() &&
|
||||||
validate_tests::run() &&
|
validate_tests::run() &&
|
||||||
minify_tests::run() &&
|
minify_tests::run() &&
|
||||||
parse_api_tests::run() &&
|
parse_api_tests::run() &&
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
#include "simdjson.h"
|
#include "simdjson.h"
|
||||||
#include "test_macros.h"
|
#include "test_macros.h"
|
||||||
|
|
||||||
void print_hex(const simdjson::padded_string& s) {
|
void print_hex(const std::string& s) {
|
||||||
printf("hex : ");
|
printf("hex : ");
|
||||||
for(size_t i = 0; i < s.size(); i++) { printf("%02X ", uint8_t(s.data()[i])); }
|
for(size_t i = 0; i < s.size(); i++) { printf("%02X ", uint8_t(s.data()[i])); }
|
||||||
printf("\n");
|
printf("\n");
|
||||||
@@ -71,7 +71,7 @@ std::string trim(const std::string s) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
namespace document_stream_tests {
|
namespace document_stream_tests {
|
||||||
static simdjson::dom::document_stream parse_many_stream_return(simdjson::dom::parser &parser, simdjson::padded_string &str) {
|
static simdjson::dom::document_stream parse_many_stream_return(simdjson::dom::parser &parser, std::string &str) {
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
simdjson_unused auto error = parser.parse_many(str).get(stream);
|
simdjson_unused auto error = parser.parse_many(str).get(stream);
|
||||||
return stream;
|
return stream;
|
||||||
@@ -79,14 +79,14 @@ namespace document_stream_tests {
|
|||||||
// this is a compilation test
|
// this is a compilation test
|
||||||
simdjson_unused static void parse_many_stream_assign() {
|
simdjson_unused static void parse_many_stream_assign() {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::padded_string str("{}",2);
|
std::string str("{}",2);
|
||||||
simdjson::dom::document_stream s1 = parse_many_stream_return(parser, str);
|
simdjson::dom::document_stream s1 = parse_many_stream_return(parser, str);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool stress_data_race() {
|
bool stress_data_race() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
// Correct JSON.
|
// Correct JSON.
|
||||||
const simdjson::padded_string input = R"([1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;;
|
const std::string input = R"([1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.parse_many(input, 32).get(stream));
|
ASSERT_SUCCESS(parser.parse_many(input, 32).get(stream));
|
||||||
@@ -103,7 +103,7 @@ namespace document_stream_tests {
|
|||||||
bool stress_data_race_with_error() {
|
bool stress_data_race_with_error() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
// Intentionally broken
|
// Intentionally broken
|
||||||
const simdjson::padded_string input = R"([1,23] [1,23] [1,23] [1,23 [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;;
|
const std::string input = R"([1,23] [1,23] [1,23] [1,23 [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.parse_many(input, 32).get(stream));
|
ASSERT_SUCCESS(parser.parse_many(input, 32).get(stream));
|
||||||
@@ -129,7 +129,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool test_leading_spaces() {
|
bool test_leading_spaces() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
const simdjson::padded_string input = R"( [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;;
|
const std::string input = R"( [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )";
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
@@ -149,7 +149,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool test_crazy_leading_spaces() {
|
bool test_crazy_leading_spaces() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
const simdjson::padded_string input = R"( [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;;
|
const std::string input = R"( [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )";
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
@@ -168,7 +168,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool issue1307() {
|
bool issue1307() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
const simdjson::padded_string input = decode_base64("AgAMACA=");
|
const std::string input = decode_base64("AgAMACA=");
|
||||||
print_hex(input);
|
print_hex(input);
|
||||||
for(size_t window = 0; window <= 100; window++) {
|
for(size_t window = 0; window <= 100; window++) {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
@@ -188,7 +188,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool issue1308() {
|
bool issue1308() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
const simdjson::padded_string input = decode_base64("bcdtW0E=");
|
const std::string input = decode_base64("bcdtW0E=");
|
||||||
print_hex(input);
|
print_hex(input);
|
||||||
for(size_t window = 0; window <= 100; window++) {
|
for(size_t window = 0; window <= 100; window++) {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
@@ -208,7 +208,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool issue1309() {
|
bool issue1309() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
const simdjson::padded_string input = decode_base64("CQA5OAo5CgoKCiIiXyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiJiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiXyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiJiIiIiIiIiIiIiIiIiIiIiLb29vb29vb29vb29vb29vz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz29vb29vb29vbIiIiIiIiIiIiIiIiIiIiIiIiIiIiJiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiYiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI=");
|
const std::string input = decode_base64("CQA5OAo5CgoKCiIiXyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiJiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiXyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiJiIiIiIiIiIiIiIiIiIiIiLb29vb29vb29vb29vb29vz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz8/Pz29vb29vb29vbIiIiIiIiIiIiIiIiIiIiIiIiIiIiJiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiYiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI=");
|
||||||
print_hex(input);
|
print_hex(input);
|
||||||
for(size_t window = 0; window <= 100; window++) {
|
for(size_t window = 0; window <= 100; window++) {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
@@ -231,7 +231,7 @@ namespace document_stream_tests {
|
|||||||
// hex : 20 20 5B 20 33 2C 31 5D 20 22 22 22 22 22 22 22 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
|
// hex : 20 20 5B 20 33 2C 31 5D 20 22 22 22 22 22 22 22 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20 20
|
||||||
// ascii: __ __[__ __3__,__1__]__ __"__"__"__"__"__"__"__ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __
|
// ascii: __ __[__ __3__,__1__]__ __"__"__"__"__"__"__"__ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __
|
||||||
// We have four full documents followed by an unclosed string.
|
// We have four full documents followed by an unclosed string.
|
||||||
const simdjson::padded_string input = decode_base64("ICBbIDMsMV0gIiIiIiIiIiAgICAgICAgICAgICAgICAg");
|
const std::string input = decode_base64("ICBbIDMsMV0gIiIiIiIiIiAgICAgICAgICAgICAgICAg");
|
||||||
print_hex(input);
|
print_hex(input);
|
||||||
for(size_t window = 0; window <= 100; window++) {
|
for(size_t window = 0; window <= 100; window++) {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
@@ -261,7 +261,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool issue1311() {
|
bool issue1311() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
const simdjson::padded_string input = decode_base64("NSMwW1swDPw=");
|
const std::string input = decode_base64("NSMwW1swDPw=");
|
||||||
print_hex(input);
|
print_hex(input);
|
||||||
for(size_t window = 0; window <= 100; window++) {
|
for(size_t window = 0; window <= 100; window++) {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
@@ -321,7 +321,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool test_naked_iterators() {
|
bool test_naked_iterators() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto json = R"([1,23] "lone string" {"key":"unfinished value} )"_padded;
|
std::string json = R"([1,23] "lone string" {"key":"unfinished value} )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
|
ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
|
||||||
@@ -344,7 +344,7 @@ namespace document_stream_tests {
|
|||||||
bool adversarial_single_document() {
|
bool adversarial_single_document() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
auto json = R"({"f[)"_padded;
|
std::string json = R"({"f[)";
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.parse_many(json).get(stream));
|
ASSERT_SUCCESS(parser.parse_many(json).get(stream));
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
@@ -359,7 +359,7 @@ namespace document_stream_tests {
|
|||||||
bool adversarial_single_document_array() {
|
bool adversarial_single_document_array() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
auto json = R"(["this is an unclosed string ])"_padded;
|
std::string json = R"(["this is an unclosed string ])";
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.parse_many(json).get(stream));
|
ASSERT_SUCCESS(parser.parse_many(json).get(stream));
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
@@ -374,7 +374,7 @@ namespace document_stream_tests {
|
|||||||
bool single_document() {
|
bool single_document() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
auto json = R"({"hello": "world"})"_padded;
|
std::string json = R"({"hello": "world"})";
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.parse_many(json).get(stream));
|
ASSERT_SUCCESS(parser.parse_many(json).get(stream));
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
@@ -401,7 +401,7 @@ namespace document_stream_tests {
|
|||||||
bool single_document_exceptions() {
|
bool single_document_exceptions() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
auto json = R"({"hello": "world"})"_padded;
|
std::string json = R"({"hello": "world"})";
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
for (simdjson::dom::element doc : parser.parse_many(json)) {
|
for (simdjson::dom::element doc : parser.parse_many(json)) {
|
||||||
std::string expected = R"({"hello":"world"})";
|
std::string expected = R"({"hello":"world"})";
|
||||||
@@ -419,7 +419,7 @@ namespace document_stream_tests {
|
|||||||
bool issue1133() {
|
bool issue1133() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
auto json = "{\"hello\": \"world\"}"_padded;
|
std::string json = "{\"hello\": \"world\"}";
|
||||||
simdjson::dom::document_stream docs = parser.parse_many(json);
|
simdjson::dom::document_stream docs = parser.parse_many(json);
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
for (simdjson::dom::element doc : docs) {
|
for (simdjson::dom::element doc : docs) {
|
||||||
@@ -441,12 +441,12 @@ namespace document_stream_tests {
|
|||||||
#endif
|
#endif
|
||||||
bool simple_example() {
|
bool simple_example() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2,3] )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
// We use a window of json.size() though any large value would do.
|
// We use a window of json.size() though any large value would do.
|
||||||
ASSERT_SUCCESS( parser.parse_many(json, json.size()).get(stream) );
|
ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
|
||||||
auto i = stream.begin();
|
auto i = stream.begin();
|
||||||
for(; i != stream.end(); ++i) {
|
for(; i != stream.end(); ++i) {
|
||||||
auto doc = *i;
|
auto doc = *i;
|
||||||
@@ -478,11 +478,11 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool unquoted_key() {
|
bool unquoted_key() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto json = R"({unquoted_key: "keys must be quoted"})"_padded;
|
std::string json = R"({unquoted_key: "keys must be quoted"})";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
// We use a window of json.size() though any large value would do.
|
// We use a window of json.size() though any large value would do.
|
||||||
ASSERT_SUCCESS( parser.parse_many(json, json.size()).get(stream) );
|
ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
|
||||||
auto i = stream.begin();
|
auto i = stream.begin();
|
||||||
for(; i != stream.end(); ++i) {
|
for(; i != stream.end(); ++i) {
|
||||||
auto doc = *i;
|
auto doc = *i;
|
||||||
@@ -503,7 +503,7 @@ namespace document_stream_tests {
|
|||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
// The last JSON document is
|
// The last JSON document is
|
||||||
// intentionally truncated.
|
// intentionally truncated.
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2 )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2 )";
|
||||||
std::cout << "input size " << json.size() << std::endl;
|
std::cout << "input size " << json.size() << std::endl;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
@@ -539,11 +539,11 @@ namespace document_stream_tests {
|
|||||||
bool truncated_window_unclosed_string() {
|
bool truncated_window_unclosed_string() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
// The last JSON document is intentionally truncated.
|
// The last JSON document is intentionally truncated.
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} "intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} "intentionally unclosed string )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
// We use a window of json.size() though any large value would do.
|
// We use a window of json.size() though any large value would do.
|
||||||
ASSERT_SUCCESS( parser.parse_many(json,json.size()).get(stream) );
|
ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
|
||||||
auto i = stream.begin();
|
auto i = stream.begin();
|
||||||
size_t counter{0};
|
size_t counter{0};
|
||||||
for(; i != stream.end(); ++i) {
|
for(; i != stream.end(); ++i) {
|
||||||
@@ -577,11 +577,11 @@ namespace document_stream_tests {
|
|||||||
bool truncated_window_unclosed_string_in_object() {
|
bool truncated_window_unclosed_string_in_object() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
// The last JSON document is intentionally truncated.
|
// The last JSON document is intentionally truncated.
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
// We use a window of json.size() though any large value would do.
|
// We use a window of json.size() though any large value would do.
|
||||||
ASSERT_SUCCESS( parser.parse_many(json,json.size()).get(stream) );
|
ASSERT_SUCCESS( parser.parse_many(json).get(stream) );
|
||||||
auto i = stream.begin();
|
auto i = stream.begin();
|
||||||
size_t counter{0};
|
size_t counter{0};
|
||||||
for(; i != stream.end(); ++i) {
|
for(; i != stream.end(); ++i) {
|
||||||
@@ -619,7 +619,7 @@ namespace document_stream_tests {
|
|||||||
input.push_back('1');
|
input.push_back('1');
|
||||||
input.push_back(i < 1023 ? ',' : ']');
|
input.push_back(i < 1023 ? ',' : ']');
|
||||||
}
|
}
|
||||||
auto json = simdjson::padded_string(input.data(),input.size());
|
auto json = std::string(input.data(),input.size());
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
size_t window_size = 1024; // deliberately too small
|
size_t window_size = 1024; // deliberately too small
|
||||||
@@ -647,7 +647,7 @@ namespace document_stream_tests {
|
|||||||
input.push_back('1');
|
input.push_back('1');
|
||||||
input.push_back(i < 1023 ? ',' : ']');
|
input.push_back(i < 1023 ? ',' : ']');
|
||||||
}
|
}
|
||||||
auto json = simdjson::padded_string(input.data(), input.size());
|
auto json = std::string(input.data(), input.size());
|
||||||
// We are going to repeat this test 1000 times so
|
// We are going to repeat this test 1000 times so
|
||||||
// that if there is an issue, we are more likely to
|
// that if there is an issue, we are more likely to
|
||||||
// trigger it systematically.
|
// trigger it systematically.
|
||||||
@@ -682,7 +682,7 @@ namespace document_stream_tests {
|
|||||||
bool large_window() {
|
bool large_window() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
#if SIZE_MAX > 17179869184
|
#if SIZE_MAX > 17179869184
|
||||||
auto json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})"_padded;
|
std::string json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
uint64_t window_size{17179869184}; // deliberately too big
|
uint64_t window_size{17179869184}; // deliberately too big
|
||||||
@@ -698,7 +698,7 @@ namespace document_stream_tests {
|
|||||||
#endif
|
#endif
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
static bool parse_json_message_issue467(simdjson::padded_string &json, size_t expectedcount) {
|
static bool parse_json_message_issue467(std::string &json, size_t expectedcount) {
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
@@ -719,8 +719,8 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool json_issue467() {
|
bool json_issue467() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
auto single_message = R"({"error":[],"result":{"token":"xxx"}})"_padded;
|
std::string single_message = R"({"error":[],"result":{"token":"xxx"}})";
|
||||||
auto two_messages = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})"_padded;
|
std::string two_messages = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})";
|
||||||
|
|
||||||
if(!parse_json_message_issue467(single_message, 1)) {
|
if(!parse_json_message_issue467(single_message, 1)) {
|
||||||
return false;
|
return false;
|
||||||
@@ -735,7 +735,7 @@ namespace document_stream_tests {
|
|||||||
bool document_stream_test() {
|
bool document_stream_test() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
fflush(NULL);
|
fflush(NULL);
|
||||||
const size_t n_records = 10000;
|
const size_t n_records = 100;
|
||||||
std::string data;
|
std::string data;
|
||||||
std::vector<char> buf(1024);
|
std::vector<char> buf(1024);
|
||||||
for (size_t i = 0; i < n_records; ++i) {
|
for (size_t i = 0; i < n_records; ++i) {
|
||||||
@@ -750,7 +750,7 @@ namespace document_stream_tests {
|
|||||||
for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) {
|
for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) {
|
||||||
printf(".");
|
printf(".");
|
||||||
fflush(NULL);
|
fflush(NULL);
|
||||||
simdjson::padded_string str(data);
|
std::string str(data);
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
@@ -775,7 +775,7 @@ namespace document_stream_tests {
|
|||||||
bool document_stream_utf8_test() {
|
bool document_stream_utf8_test() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
fflush(NULL);
|
fflush(NULL);
|
||||||
const size_t n_records = 10000;
|
const size_t n_records = 100;
|
||||||
std::string data;
|
std::string data;
|
||||||
std::vector<char> buf(1024);
|
std::vector<char> buf(1024);
|
||||||
for (size_t i = 0; i < n_records; ++i) {
|
for (size_t i = 0; i < n_records; ++i) {
|
||||||
@@ -790,7 +790,7 @@ namespace document_stream_tests {
|
|||||||
for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) {
|
for(size_t batch_size = 1000; batch_size < 2000; batch_size += (batch_size>1050?10:1)) {
|
||||||
printf(".");
|
printf(".");
|
||||||
fflush(NULL);
|
fflush(NULL);
|
||||||
simdjson::padded_string str(data);
|
std::string str(data);
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
size_t count = 0;
|
size_t count = 0;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
@@ -811,7 +811,7 @@ namespace document_stream_tests {
|
|||||||
bool issue1649() {
|
bool issue1649() {
|
||||||
std::cout << "Running " << __func__ << std::endl;
|
std::cout << "Running " << __func__ << std::endl;
|
||||||
std::size_t batch_size = 637;
|
std::size_t batch_size = 637;
|
||||||
const auto json=simdjson::padded_string(std::string("\xd7"));
|
const auto json=std::string("\xd7");
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream docs;
|
simdjson::dom::document_stream docs;
|
||||||
if(parser.parse_many(json,batch_size).get(docs)) {
|
if(parser.parse_many(json,batch_size).get(docs)) {
|
||||||
@@ -838,7 +838,7 @@ namespace document_stream_tests {
|
|||||||
for(size_t z = 0; z < l; z++) {
|
for(size_t z = 0; z < l; z++) {
|
||||||
buffer[z] = char(ascii(gen));
|
buffer[z] = char(ascii(gen));
|
||||||
}
|
}
|
||||||
const auto json = simdjson::padded_string(buffer, l);
|
const auto json = std::string(buffer, l);
|
||||||
delete[] buffer;
|
delete[] buffer;
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream docs;
|
simdjson::dom::document_stream docs;
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ namespace document_tests {
|
|||||||
std::vector<std::string> json_strings{"[true,false]", "[1,2,3,null]",
|
std::vector<std::string> json_strings{"[true,false]", "[1,2,3,null]",
|
||||||
R"({"yay":"json!"})"};
|
R"({"yay":"json!"})"};
|
||||||
simdjson::dom::parser parser1;
|
simdjson::dom::parser parser1;
|
||||||
for (simdjson::padded_string str : json_strings) {
|
for (std::string str : json_strings) {
|
||||||
simdjson::dom::element element;
|
simdjson::dom::element element;
|
||||||
ASSERT_SUCCESS( parser1.parse(str).get(element) );
|
ASSERT_SUCCESS( parser1.parse(str).get(element) );
|
||||||
std::cout << element << std::endl;
|
std::cout << element << std::endl;
|
||||||
@@ -39,14 +39,14 @@ namespace document_tests {
|
|||||||
// adversarial example that once triggered overruns, see https://github.com/simdjson/simdjson/issues/345
|
// adversarial example that once triggered overruns, see https://github.com/simdjson/simdjson/issues/345
|
||||||
bool bad_example() {
|
bool bad_example() {
|
||||||
std::cout << __func__ << std::endl;
|
std::cout << __func__ << std::endl;
|
||||||
simdjson::padded_string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6"_padded;
|
std::string badjson = "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
ASSERT_ERROR( parser.parse(badjson), simdjson::TAPE_ERROR );
|
ASSERT_ERROR( parser.parse(badjson), simdjson::TAPE_ERROR );
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
bool count_array_example() {
|
bool count_array_example() {
|
||||||
std::cout << __func__ << std::endl;
|
std::cout << __func__ << std::endl;
|
||||||
simdjson::padded_string smalljson = "[1,2,3]"_padded;
|
std::string smalljson = "[1,2,3]";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::array array;
|
simdjson::dom::array array;
|
||||||
ASSERT_SUCCESS( parser.parse(smalljson).get(array) );
|
ASSERT_SUCCESS( parser.parse(smalljson).get(array) );
|
||||||
@@ -55,7 +55,7 @@ namespace document_tests {
|
|||||||
}
|
}
|
||||||
bool count_object_example() {
|
bool count_object_example() {
|
||||||
std::cout << __func__ << std::endl;
|
std::cout << __func__ << std::endl;
|
||||||
simdjson::padded_string smalljson = "{\"1\":1,\"2\":1,\"3\":1}"_padded;
|
std::string smalljson = "{\"1\":1,\"2\":1,\"3\":1}";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::object object;
|
simdjson::dom::object object;
|
||||||
ASSERT_SUCCESS( parser.parse(smalljson).get(object) );
|
ASSERT_SUCCESS( parser.parse(smalljson).get(object) );
|
||||||
@@ -74,7 +74,7 @@ namespace document_tests {
|
|||||||
// returns true if successful
|
// returns true if successful
|
||||||
bool stable_test() {
|
bool stable_test() {
|
||||||
std::cout << __func__ << std::endl;
|
std::cout << __func__ << std::endl;
|
||||||
simdjson::padded_string json = "{"
|
std::string json = "{"
|
||||||
"\"Image\":{"
|
"\"Image\":{"
|
||||||
"\"Width\":800,"
|
"\"Width\":800,"
|
||||||
"\"Height\":600,"
|
"\"Height\":600,"
|
||||||
@@ -87,7 +87,7 @@ namespace document_tests {
|
|||||||
"\"Animated\":false,"
|
"\"Animated\":false,"
|
||||||
"\"IDs\":[116,943.3,234,38793]"
|
"\"IDs\":[116,943.3,234,38793]"
|
||||||
"}"
|
"}"
|
||||||
"}"_padded;
|
"}";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
std::ostringstream myStream;
|
std::ostringstream myStream;
|
||||||
#if SIMDJSON_EXCEPTIONS
|
#if SIMDJSON_EXCEPTIONS
|
||||||
|
|||||||
@@ -182,7 +182,7 @@ bool issue1142() {
|
|||||||
|
|
||||||
const char * input_array = "[]";
|
const char * input_array = "[]";
|
||||||
size_t input_length = std::strlen(input_array);
|
size_t input_length = std::strlen(input_array);
|
||||||
auto element4 = parser.parse(input_array, input_length).at_pointer("");;
|
auto element4 = parser.parse(input_array, input_length).at_pointer("");
|
||||||
ASSERT_EQUAL(std::string(R"([])"), simdjson::minify(element4));
|
ASSERT_EQUAL(std::string(R"([])"), simdjson::minify(element4));
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ bool tester(int seed, size_t volume) {
|
|||||||
int main(int argc, char *argv[]) {
|
int main(int argc, char *argv[]) {
|
||||||
// We test 1,000,000 random strings by default.
|
// We test 1,000,000 random strings by default.
|
||||||
// You can specify more tests with the '-m' flag if you want.
|
// You can specify more tests with the '-m' flag if you want.
|
||||||
size_t howmany = 1000000;
|
size_t howmany = 10000;
|
||||||
|
|
||||||
int c;
|
int c;
|
||||||
while ((c = getopt(argc, argv, "a:m:h")) != -1) {
|
while ((c = getopt(argc, argv, "a:m:h")) != -1) {
|
||||||
|
|||||||
@@ -16,17 +16,17 @@ void basics_1() {
|
|||||||
|
|
||||||
void basics_2() {
|
void basics_2() {
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::element doc = parser.parse("[1,2,3]"_padded); // parse a string
|
dom::element doc = parser.parse(std::string("[1,2,3]")); // parse a string
|
||||||
|
|
||||||
cout << doc;
|
cout << doc;
|
||||||
}
|
}
|
||||||
|
|
||||||
void basics_dom_1() {
|
void basics_dom_1() {
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
// Parse and iterate through each car
|
// Parse and iterate through each car
|
||||||
@@ -53,7 +53,7 @@ void basics_dom_1() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void parse_many_truncated() {
|
void parse_many_truncated() {
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )";
|
||||||
simdjson::dom::parser parser;
|
simdjson::dom::parser parser;
|
||||||
simdjson::dom::document_stream stream;
|
simdjson::dom::document_stream stream;
|
||||||
auto error = parser.parse_many(json,json.size()).get(stream);
|
auto error = parser.parse_many(json,json.size()).get(stream);
|
||||||
@@ -65,11 +65,11 @@ void parse_many_truncated() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void basics_dom_2() {
|
void basics_dom_2() {
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::element cars = parser.parse(cars_json);
|
dom::element cars = parser.parse(cars_json);
|
||||||
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
cout << cars.at_pointer("/0/tire_pressure/1") << endl; // Prints 39.9
|
||||||
@@ -83,10 +83,10 @@ void basics_dom_2() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void basics_dom_3() {
|
void basics_dom_3() {
|
||||||
auto abstract_json = R"( [
|
std::string abstract_json = R"( [
|
||||||
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
{ "12345" : {"a":12.34, "b":56.78, "c": 9998877} },
|
||||||
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
{ "12545" : {"a":11.44, "b":12.78, "c": 11111111} }
|
||||||
] )"_padded;
|
] )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
// Parse and iterate through an array of objects
|
// Parse and iterate through an array of objects
|
||||||
@@ -102,8 +102,8 @@ void basics_dom_3() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void basics_dom_4() {
|
void basics_dom_4() {
|
||||||
auto abstract_json = R"(
|
std::string abstract_json = R"(
|
||||||
{ "str" : { "123" : {"abc" : 3.14 } } } )"_padded;
|
{ "str" : { "123" : {"abc" : 3.14 } } } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
|
double v = parser.parse(abstract_json)["str"]["123"]["abc"];
|
||||||
cout << "number: " << v << endl;
|
cout << "number: " << v << endl;
|
||||||
@@ -221,7 +221,7 @@ namespace treewalk_1 {
|
|||||||
|
|
||||||
#ifdef SIMDJSON_CPLUSPLUS17
|
#ifdef SIMDJSON_CPLUSPLUS17
|
||||||
void basics_cpp17_1() {
|
void basics_cpp17_1() {
|
||||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
std::string json = R"( { "foo": 1, "bar": 2 } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::object object;
|
dom::object object;
|
||||||
auto error = parser.parse(json).get(object);
|
auto error = parser.parse(json).get(object);
|
||||||
@@ -234,7 +234,7 @@ void basics_cpp17_1() {
|
|||||||
|
|
||||||
void basics_cpp17_2() {
|
void basics_cpp17_2() {
|
||||||
// C++ 11 version for comparison
|
// C++ 11 version for comparison
|
||||||
padded_string json = R"( { "foo": 1, "bar": 2 } )"_padded;
|
std::string json = R"( { "foo": 1, "bar": 2 } )";
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
dom::object object;
|
dom::object object;
|
||||||
auto error = parser.parse(json).get(object);
|
auto error = parser.parse(json).get(object);
|
||||||
@@ -254,9 +254,9 @@ void basics_ndjson() {
|
|||||||
|
|
||||||
void basics_ndjson_parse_many() {
|
void basics_ndjson_parse_many() {
|
||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
auto json = R"({ "foo": 1 }
|
std::string json = R"({ "foo": 1 }
|
||||||
{ "foo": 2 }
|
{ "foo": 2 }
|
||||||
{ "foo": 3 })"_padded;
|
{ "foo": 3 })";
|
||||||
dom::document_stream docs = parser.parse_many(json);
|
dom::document_stream docs = parser.parse_many(json);
|
||||||
for (dom::element doc : docs) {
|
for (dom::element doc : docs) {
|
||||||
cout << doc["foo"] << endl;
|
cout << doc["foo"] << endl;
|
||||||
@@ -269,7 +269,7 @@ void implementation_selection_1() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
void unescaped_key() {
|
void unescaped_key() {
|
||||||
auto json = R"({"k\u0065y": 1})"_padded;
|
std::string json = R"({"k\u0065y": 1})";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
ondemand::object object = doc.get_object();
|
ondemand::object object = doc.get_object();
|
||||||
@@ -320,7 +320,7 @@ void ondemand_performance_1() {
|
|||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
|
|
||||||
// This initializes buffers big enough to handle this JSON.
|
// This initializes buffers big enough to handle this JSON.
|
||||||
auto json = "[ true, false ]"_padded;
|
std::string json = "[ true, false ]";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
for(bool i : doc.get_array()) {
|
for(bool i : doc.get_array()) {
|
||||||
cout << i << endl;
|
cout << i << endl;
|
||||||
@@ -328,7 +328,7 @@ void ondemand_performance_1() {
|
|||||||
|
|
||||||
|
|
||||||
// This reuses the existing buffers
|
// This reuses the existing buffers
|
||||||
auto number_json = "[1, 2, 3]"_padded;
|
std::string number_json = "[1, 2, 3]";
|
||||||
doc = parser.iterate(number_json);
|
doc = parser.iterate(number_json);
|
||||||
for(int64_t i : doc.get_array()) {
|
for(int64_t i : doc.get_array()) {
|
||||||
cout << i << endl;
|
cout << i << endl;
|
||||||
@@ -339,15 +339,15 @@ void performance_1() {
|
|||||||
dom::parser parser;
|
dom::parser parser;
|
||||||
|
|
||||||
// This initializes buffers and a document big enough to handle this JSON.
|
// This initializes buffers and a document big enough to handle this JSON.
|
||||||
dom::element doc = parser.parse("[ true, false ]"_padded);
|
dom::element doc = parser.parse(std::string("[ true, false ]"));
|
||||||
cout << doc << endl;
|
cout << doc << endl;
|
||||||
|
|
||||||
// This reuses the existing buffers, and reuses and *overwrites* the old document
|
// This reuses the existing buffers, and reuses and *overwrites* the old document
|
||||||
doc = parser.parse("[1, 2, 3]"_padded);
|
doc = parser.parse(std::string("[1, 2, 3]"));
|
||||||
cout << doc << endl;
|
cout << doc << endl;
|
||||||
|
|
||||||
// This also reuses the existing buffers, and reuses and *overwrites* the old document
|
// This also reuses the existing buffers, and reuses and *overwrites* the old document
|
||||||
dom::element doc2 = parser.parse("true"_padded);
|
dom::element doc2 = parser.parse(std::string("true"));
|
||||||
// Even if you keep the old reference around, doc and doc2 refer to the same document.
|
// Even if you keep the old reference around, doc and doc2 refer to the same document.
|
||||||
cout << doc << endl;
|
cout << doc << endl;
|
||||||
cout << doc2 << endl;
|
cout << doc2 << endl;
|
||||||
@@ -360,7 +360,7 @@ void performance_2() {
|
|||||||
dom::parser parser(1000*1000); // Never grow past documents > 1MB
|
dom::parser parser(1000*1000); // Never grow past documents > 1MB
|
||||||
/* for (web_request request : listen()) */ {
|
/* for (web_request request : listen()) */ {
|
||||||
dom::element doc;
|
dom::element doc;
|
||||||
auto body = "1"_padded; /*request.body*/
|
std::string body = "1"; /*request.body*/
|
||||||
auto error = parser.parse(body/*request.body*/).get(doc);
|
auto error = parser.parse(body/*request.body*/).get(doc);
|
||||||
// If the document was above our limit, emit 413 = payload too large
|
// If the document was above our limit, emit 413 = payload too large
|
||||||
if (error == CAPACITY) { /* request.respond(413); continue; */ }
|
if (error == CAPACITY) { /* request.respond(413); continue; */ }
|
||||||
@@ -376,7 +376,7 @@ void performance_3() {
|
|||||||
|
|
||||||
/* for (web_request request : listen()) */ {
|
/* for (web_request request : listen()) */ {
|
||||||
dom::element doc;
|
dom::element doc;
|
||||||
auto body = "1"_padded;/*request.body*/
|
std::string body = "1";/*request.body*/
|
||||||
auto error = parser.parse(body).get(doc);
|
auto error = parser.parse(body).get(doc);
|
||||||
// If the document was above our limit, emit 413 = payload too large
|
// If the document was above our limit, emit 413 = payload too large
|
||||||
if (error == CAPACITY) { /* request.respond(413); continue; */ }
|
if (error == CAPACITY) { /* request.respond(413); continue; */ }
|
||||||
@@ -463,5 +463,6 @@ int main() {
|
|||||||
basics_dom_3();
|
basics_dom_3();
|
||||||
basics_dom_4();
|
basics_dom_4();
|
||||||
minify();
|
minify();
|
||||||
return 0;
|
printf("Tests ok.\n");
|
||||||
|
return EXIT_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
link_libraries(simdjson)
|
link_libraries(simdjson)
|
||||||
include_directories(..)
|
include_directories(..)
|
||||||
add_subdirectory(compilation_failure_tests)
|
add_subdirectory(compilation_failure_tests)
|
||||||
add_cpp_test(ondemand_tostring_tests LABELS ondemand acceptance per_implementation)
|
|
||||||
add_cpp_test(ondemand_active_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_active_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_array_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_array_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_array_error_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_array_error_tests LABELS ondemand acceptance per_implementation)
|
||||||
@@ -15,10 +15,12 @@ add_cpp_test(ondemand_misc_tests LABELS ondemand acceptance per_impl
|
|||||||
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
|
||||||
|
add_cpp_test(ondemand_object_find_field_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_readme_examples LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_readme_examples LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_scalar_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_scalar_tests LABELS ondemand acceptance per_implementation)
|
||||||
|
add_cpp_test(ondemand_tostring_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
|
||||||
|
|
||||||
|
|||||||
@@ -15,5 +15,5 @@ endfunction(add_dual_compile_test)
|
|||||||
|
|
||||||
|
|
||||||
add_dual_compile_test(iterate_char_star)
|
add_dual_compile_test(iterate_char_star)
|
||||||
add_dual_compile_test(iterate_string_view)
|
|
||||||
add_dual_compile_test(iterate_temporary_buffer)
|
add_dual_compile_test(iterate_temporary_buffer)
|
||||||
|
add_dual_compile_test(padded_string_view_char_star_no_capacity)
|
||||||
|
|||||||
@@ -6,10 +6,9 @@ using namespace simdjson;
|
|||||||
int main() {
|
int main() {
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
#if COMPILATION_TEST_USE_FAILING_CODE
|
#if COMPILATION_TEST_USE_FAILING_CODE
|
||||||
const char* json;
|
auto doc = parser.iterate("1");
|
||||||
auto doc = parser.iterate(json, strlen(json));
|
|
||||||
#else
|
#else
|
||||||
auto json = "1"_padded;
|
std::string json = "1";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
#endif
|
#endif
|
||||||
int64_t value;
|
int64_t value;
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ using namespace simdjson;
|
|||||||
int main() {
|
int main() {
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
#if COMPILATION_TEST_USE_FAILING_CODE
|
#if COMPILATION_TEST_USE_FAILING_CODE
|
||||||
auto doc = parser.iterate("1"_padded);
|
auto doc = parser.iterate(std::string("1"));
|
||||||
#else
|
#else
|
||||||
auto json = "1"_padded;
|
std::string json = "1";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
#endif
|
#endif
|
||||||
int64_t value;
|
int64_t value;
|
||||||
|
|||||||
+7
-1
@@ -4,12 +4,18 @@
|
|||||||
using namespace simdjson;
|
using namespace simdjson;
|
||||||
|
|
||||||
int main() {
|
int main() {
|
||||||
|
auto json_chars = "1 ";
|
||||||
|
#if COMPILATION_TEST_USE_FAILING_CODE
|
||||||
|
simdjson_unused padded_string_view p{json_chars, 1};
|
||||||
|
#else
|
||||||
|
simdjson_unused padded_string_view p{json_chars, 1, strlen(json_chars)};
|
||||||
|
#endif
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
#if COMPILATION_TEST_USE_FAILING_CODE
|
#if COMPILATION_TEST_USE_FAILING_CODE
|
||||||
auto json = std::string_view("1");
|
auto json = std::string_view("1");
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
#else
|
#else
|
||||||
auto json = "1"_padded;
|
std::string json = "1";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
#endif
|
#endif
|
||||||
int64_t value;
|
int64_t value;
|
||||||
@@ -10,7 +10,7 @@ namespace active_tests {
|
|||||||
bool parser_child() {
|
bool parser_child() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
const padded_string json = R"({ "parent": {"child1": {"name": "John"} , "child2": {"name": "Daniel"}} })"_padded;
|
const std::string json = R"({ "parent": {"child1": {"name": "John"} , "child2": {"name": "Daniel"}} })";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
ondemand::object parent = doc["parent"];
|
ondemand::object parent = doc["parent"];
|
||||||
{
|
{
|
||||||
@@ -27,7 +27,7 @@ namespace active_tests {
|
|||||||
bool parser_doc_correct() {
|
bool parser_doc_correct() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
const padded_string json = R"({ "key1": 1, "key2":2, "key3": 3 })"_padded;
|
const std::string json = R"({ "key1": 1, "key2":2, "key3": 3 })";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
ondemand::object root_object = doc.get_object();
|
ondemand::object root_object = doc.get_object();
|
||||||
int64_t k1 = root_object["key1"];
|
int64_t k1 = root_object["key1"];
|
||||||
@@ -39,7 +39,7 @@ namespace active_tests {
|
|||||||
bool parser_doc_limits() {
|
bool parser_doc_limits() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
const padded_string json = R"({ "key1": 1, "key2":2, "key3": 3 })"_padded;
|
const std::string json = R"({ "key1": 1, "key2":2, "key3": 3 })";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
int64_t k1 = doc["key1"];
|
int64_t k1 = doc["key1"];
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -100,17 +100,23 @@ namespace array_error_tests {
|
|||||||
}
|
}
|
||||||
bool array_iterate_incomplete_error() {
|
bool array_iterate_incomplete_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
|
#if SIMDJSON_CHECK_EOF
|
||||||
ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||||
ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
|
||||||
ONDEMAND_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||||
ONDEMAND_SUBTEST("unclosed after array", R"([ [])", assert_iterate(doc.get_array().at(0), { INCOMPLETE_ARRAY_OR_OBJECT }));
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [])", assert_iterate(doc.get_array().at(0), { INCOMPLETE_ARRAY_OR_OBJECT }));
|
||||||
|
#else
|
||||||
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }));
|
||||||
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR }));
|
||||||
|
ONDEMAND_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) }));
|
||||||
|
#endif
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}
|
}
|
||||||
|
|
||||||
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
||||||
bool out_of_order_array_iteration_error() {
|
bool out_of_order_array_iteration_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ [ 1, 2 ] ])"_padded;
|
std::string json = R"([ [ 1, 2 ] ])";
|
||||||
SUBTEST("simdjson_result<value>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<value>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
for (auto arr : doc) {
|
for (auto arr : doc) {
|
||||||
for (auto subelement : arr) { ASSERT_SUCCESS(subelement); }
|
for (auto subelement : arr) { ASSERT_SUCCESS(subelement); }
|
||||||
@@ -149,7 +155,7 @@ namespace array_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_top_level_array_iteration_error() {
|
bool out_of_order_top_level_array_iteration_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ 1, 2 ])"_padded;
|
std::string json = R"([ 1, 2 ])";
|
||||||
SUBTEST("simdjson_result<document>", test_ondemand_doc(json, [&](auto arr) {
|
SUBTEST("simdjson_result<document>", test_ondemand_doc(json, [&](auto arr) {
|
||||||
for (auto element : arr) { ASSERT_SUCCESS(element); }
|
for (auto element : arr) { ASSERT_SUCCESS(element); }
|
||||||
ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION );
|
ASSERT_ITERATE_ERROR( arr, OUT_OF_ORDER_ITERATION );
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ namespace array_tests {
|
|||||||
using simdjson::ondemand::json_type;
|
using simdjson::ondemand::json_type;
|
||||||
bool issue1588() {
|
bool issue1588() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
const auto json = R"({
|
const std::string json = R"({
|
||||||
"nodes" : [
|
"nodes" : [
|
||||||
{
|
{
|
||||||
"rotation" : [
|
"rotation" : [
|
||||||
@@ -51,7 +51,7 @@ namespace array_tests {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
})"_padded;
|
})";
|
||||||
// we query 'rotation', 'scale', 'translation' in sequence
|
// we query 'rotation', 'scale', 'translation' in sequence
|
||||||
const bool expected_value[][3] = { {true, false, true},
|
const bool expected_value[][3] = { {true, false, true},
|
||||||
{true, false, false}, {false, false, true}, {false, true, false} };
|
{true, false, false}, {false, false, true}, {false, true, false} };
|
||||||
@@ -131,7 +131,7 @@ namespace array_tests {
|
|||||||
bool iterate_complex_array_count() {
|
bool iterate_complex_array_count() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto cars_json = R"( { "zero":[], "test":[ { "val1":1, "val2":2 }, { "val1":1, "val2":2 } ] } )"_padded;
|
std::string cars_json = R"( { "zero":[], "test":[ { "val1":1, "val2":2 }, { "val1":1, "val2":2 } ] } )";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(cars_json).get(doc));
|
ASSERT_SUCCESS(parser.iterate(cars_json).get(doc));
|
||||||
ondemand::array firstmyarray;
|
ondemand::array firstmyarray;
|
||||||
@@ -155,7 +155,7 @@ namespace array_tests {
|
|||||||
bool iterate_sub_array_count() {
|
bool iterate_sub_array_count() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto key_value_json = R"( { "test":[ 1,2,3], "joe": [1,2] } )"_padded;
|
std::string key_value_json = R"( { "test":[ 1,2,3], "joe": [1,2] } )";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(key_value_json).get(doc));
|
ASSERT_SUCCESS(parser.iterate(key_value_json).get(doc));
|
||||||
ondemand::object obj;
|
ondemand::object obj;
|
||||||
@@ -173,8 +173,8 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_array_count() {
|
bool iterate_array_count() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
const auto json = R"([ 1, 10, 100 ])"_padded;
|
const std::string json = R"([ 1, 10, 100 ])";
|
||||||
const auto badjson = R"([ 1, 10 100 ])"_padded;
|
const std::string badjson = R"([ 1, 10 100 ])";
|
||||||
const vector<uint64_t> expected_value = { 1, 10, 100 };
|
const vector<uint64_t> expected_value = { 1, 10, 100 };
|
||||||
|
|
||||||
SUBTEST("ondemand::count_elements", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::count_elements", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
@@ -209,7 +209,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_bad_array_count() {
|
bool iterate_bad_array_count() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
const auto badjson = R"([ 1, 10 100 ])"_padded;
|
const std::string badjson = R"([ 1, 10 100 ])";
|
||||||
|
|
||||||
|
|
||||||
SUBTEST("ondemand::count_elements", test_ondemand_doc(badjson, [&](auto doc_result) {
|
SUBTEST("ondemand::count_elements", test_ondemand_doc(badjson, [&](auto doc_result) {
|
||||||
@@ -230,7 +230,7 @@ namespace array_tests {
|
|||||||
}
|
}
|
||||||
bool iterate_document_array() {
|
bool iterate_document_array() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
const auto json = R"([ 1, 10, 100 ])"_padded;
|
const std::string json = R"([ 1, 10, 100 ])";
|
||||||
const uint64_t expected_value[] = { 1, 10, 100 };
|
const uint64_t expected_value[] = { 1, 10, 100 };
|
||||||
|
|
||||||
SUBTEST("ondemand::array", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::array", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
@@ -306,7 +306,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_array() {
|
bool iterate_array() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
const auto json = R"( [ [ 1, 10, 100 ] ] )"_padded;
|
const std::string json = R"( [ [ 1, 10, 100 ] ] )";
|
||||||
const uint64_t expected_value[] = { 1, 10, 100 };
|
const uint64_t expected_value[] = { 1, 10, 100 };
|
||||||
|
|
||||||
SUBTEST("ondemand::array", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::array", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
@@ -381,7 +381,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_array_partial_children() {
|
bool iterate_array_partial_children() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"(
|
std::string json = R"(
|
||||||
[
|
[
|
||||||
0,
|
0,
|
||||||
[],
|
[],
|
||||||
@@ -395,7 +395,7 @@ namespace array_tests {
|
|||||||
{ "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 },
|
{ "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 },
|
||||||
10
|
10
|
||||||
]
|
]
|
||||||
)"_padded;
|
)";
|
||||||
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
size_t i = 0;
|
size_t i = 0;
|
||||||
for (auto value : doc_result) {
|
for (auto value : doc_result) {
|
||||||
@@ -518,7 +518,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_empty_array() {
|
bool iterate_empty_array() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = "[]"_padded;
|
std::string json = "[]";
|
||||||
SUBTEST("ondemand::array", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::array", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::array array;
|
ondemand::array array;
|
||||||
ASSERT_SUCCESS( doc_result.get(array) );
|
ASSERT_SUCCESS( doc_result.get(array) );
|
||||||
@@ -563,7 +563,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_array_exception() {
|
bool iterate_array_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ 1, 10, 100 ])"_padded;
|
std::string json = R"([ 1, 10, 100 ])";
|
||||||
const uint64_t expected_value[] = { 1, 10, 100 };
|
const uint64_t expected_value[] = { 1, 10, 100 };
|
||||||
|
|
||||||
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
@@ -577,7 +577,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_empty_object_exception() {
|
bool iterate_empty_object_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({})"_padded;
|
std::string json = R"({})";
|
||||||
|
|
||||||
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
for (simdjson_unused ondemand::field field : doc_result.get_object()) {
|
for (simdjson_unused ondemand::field field : doc_result.get_object()) {
|
||||||
@@ -591,7 +591,7 @@ namespace array_tests {
|
|||||||
|
|
||||||
bool iterate_empty_array_exception() {
|
bool iterate_empty_array_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = "[]"_padded;
|
std::string json = "[]";
|
||||||
|
|
||||||
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
for (simdjson_unused ondemand::value value : doc_result) { TEST_FAIL("Unexpected value"); }
|
for (simdjson_unused ondemand::value value : doc_result) { TEST_FAIL("Unexpected value"); }
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ simdjson_never_inline bool check_point(simdjson_result<ondemand::value> xval, si
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool test_check_point() {
|
bool test_check_point() {
|
||||||
auto json = R"(
|
std::string json = R"(
|
||||||
{
|
{
|
||||||
"x": 1,
|
"x": 1,
|
||||||
"y": 2 3
|
"y": 2 3
|
||||||
)"_padded;
|
)";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
return check_point(doc["x"], doc["y"]);
|
return check_point(doc["x"], doc["y"]);
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ void process3(int ) {}
|
|||||||
|
|
||||||
// Do not run this, it is only meant to compile
|
// Do not run this, it is only meant to compile
|
||||||
void compilation_test_1() {
|
void compilation_test_1() {
|
||||||
const padded_string bogus = ""_padded;
|
const std::string bogus = "";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(bogus);
|
auto doc = parser.iterate(bogus);
|
||||||
for (ondemand::object my_object : doc["mykey"]) {
|
for (ondemand::object my_object : doc["mykey"]) {
|
||||||
@@ -27,7 +27,7 @@ void compilation_test_1() {
|
|||||||
|
|
||||||
// Do not run this, it is only meant to compile
|
// Do not run this, it is only meant to compile
|
||||||
void compilation_test_2() {
|
void compilation_test_2() {
|
||||||
const padded_string bogus = ""_padded;
|
const std::string bogus = "";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(bogus);
|
auto doc = parser.iterate(bogus);
|
||||||
std::set<std::string_view> default_users;
|
std::set<std::string_view> default_users;
|
||||||
@@ -44,7 +44,7 @@ void compilation_test_1() {
|
|||||||
|
|
||||||
// Do not run this, it is only meant to compile
|
// Do not run this, it is only meant to compile
|
||||||
void compilation_test_3() {
|
void compilation_test_3() {
|
||||||
const padded_string bogus = ""_padded;
|
const std::string bogus = "";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto doc = parser.iterate(bogus);
|
auto doc = parser.iterate(bogus);
|
||||||
ondemand::array tweets;
|
ondemand::array tweets;
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool simple_document_iteration() {
|
bool simple_document_iteration() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])"_padded;
|
std::string json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -29,7 +29,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool simple_document_iteration_multiple_batches() {
|
bool simple_document_iteration_multiple_batches() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])"_padded;
|
std::string json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json,32).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json,32).get(stream));
|
||||||
@@ -45,7 +45,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool simple_document_iteration_with_parsing() {
|
bool simple_document_iteration_with_parsing() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])"_padded;
|
std::string json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3])";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -81,7 +81,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool atoms_json() {
|
bool atoms_json() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"(5 true 20.3 "string" )"_padded;
|
std::string json = R"(5 true 20.3 "string" )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -97,7 +97,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool doc_index() {
|
bool doc_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311])"_padded;
|
std::string json = R"({"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311])";
|
||||||
std::string_view expected[5] = {R"({"z":5})",R"({"1":1,"2":2,"4":4})","[7, 10, 9]","[15, 11, 12, 13]","[154, 110, 112, 1311]"};
|
std::string_view expected[5] = {R"({"z":5})",R"({"1":1,"2":2,"4":4})","[7, 10, 9]","[15, 11, 12, 13]","[154, 110, 112, 1311]"};
|
||||||
size_t expected_indexes[5] = {0, 9, 29, 44, 65};
|
size_t expected_indexes[5] = {0, 9, 29, 44, 65};
|
||||||
|
|
||||||
@@ -117,7 +117,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool doc_index_multiple_batches() {
|
bool doc_index_multiple_batches() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311])"_padded;
|
std::string json = R"({"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311])";
|
||||||
std::string_view expected[5] = {R"({"z":5})",R"({"1":1,"2":2,"4":4})","[7, 10, 9]","[15, 11, 12, 13]","[154, 110, 112, 1311]"};
|
std::string_view expected[5] = {R"({"z":5})",R"({"1":1,"2":2,"4":4})","[7, 10, 9]","[15, 11, 12, 13]","[154, 110, 112, 1311]"};
|
||||||
size_t expected_indexes[5] = {0, 9, 29, 44, 65};
|
size_t expected_indexes[5] = {0, 9, 29, 44, 65};
|
||||||
|
|
||||||
@@ -137,7 +137,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool source_test() {
|
bool source_test() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3] )"_padded;
|
std::string json = R"([1,[1,2]] {"a":1,"b":2} {"o":{"1":1,"2":2}} [1,2,3] )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -153,7 +153,7 @@ namespace document_stream_tests {
|
|||||||
bool truncated() {
|
bool truncated() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
// The last JSON document is intentionally truncated.
|
// The last JSON document is intentionally truncated.
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2 )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2 )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -170,7 +170,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool truncated_complete_docs() {
|
bool truncated_complete_docs() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2] )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} [1,2] )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -188,7 +188,7 @@ namespace document_stream_tests {
|
|||||||
bool truncated_unclosed_string() {
|
bool truncated_unclosed_string() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
// The last JSON document is intentionally truncated.
|
// The last JSON document is intentionally truncated.
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} "intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} "intentionally unclosed string )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
// We use a window of json.size() though any large value would do.
|
// We use a window of json.size() though any large value would do.
|
||||||
@@ -205,7 +205,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool truncated_unclosed_string_in_object() {
|
bool truncated_unclosed_string_in_object() {
|
||||||
// The last JSON document is intentionally truncated.
|
// The last JSON document is intentionally truncated.
|
||||||
auto json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )"_padded;
|
std::string json = R"([1,2,3] {"1":1,"2":3,"4":4} {"key":"intentionally unclosed string )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS( parser.iterate_many(json).get(stream) );
|
ASSERT_SUCCESS( parser.iterate_many(json).get(stream) );
|
||||||
@@ -240,7 +240,7 @@ namespace document_stream_tests {
|
|||||||
bool large_window() {
|
bool large_window() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
#if SIZE_MAX > 17179869184
|
#if SIZE_MAX > 17179869184
|
||||||
auto json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})"_padded;
|
std::string json = R"({"error":[],"result":{"token":"xxx"}}{"error":[],"result":{"token":"xxx"}})";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
uint64_t window_size{17179869184}; // deliberately too big
|
uint64_t window_size{17179869184}; // deliberately too big
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
@@ -253,7 +253,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool test_leading_spaces() {
|
bool test_leading_spaces() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto input = R"( [1,1] [1,2] [1,3] [1,4] [1,5] [1,6] [1,7] [1,8] [1,9] [1,10] [1,11] [1,12] [1,13] [1,14] [1,15] )"_padded;;
|
std::string input = R"( [1,1] [1,2] [1,3] [1,4] [1,5] [1,6] [1,7] [1,8] [1,9] [1,10] [1,11] [1,12] [1,13] [1,14] [1,15] )";
|
||||||
size_t count{0};
|
size_t count{0};
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
@@ -269,7 +269,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool test_crazy_leading_spaces() {
|
bool test_crazy_leading_spaces() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto input = R"( [1,1] [1,2] [1,3] [1,4] [1,5] [1,6] [1,7] [1,8] [1,9] [1,10] [1,11] [1,12] [1,13] [1,14] [1,15] )"_padded;;
|
std::string input = R"( [1,1] [1,2] [1,3] [1,4] [1,5] [1,6] [1,7] [1,8] [1,9] [1,10] [1,11] [1,12] [1,13] [1,14] [1,15] )";
|
||||||
size_t count{0};
|
size_t count{0};
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
@@ -284,7 +284,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool adversarial_single_document() {
|
bool adversarial_single_document() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({"f[)"_padded;
|
std::string json = R"({"f[)";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -299,7 +299,7 @@ namespace document_stream_tests {
|
|||||||
|
|
||||||
bool adversarial_single_document_array() {
|
bool adversarial_single_document_array() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"(["this is an unclosed string ])"_padded;
|
std::string json = R"(["this is an unclosed string ])";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(json).get(stream));
|
||||||
@@ -315,7 +315,7 @@ namespace document_stream_tests {
|
|||||||
bool document_stream_test() {
|
bool document_stream_test() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
fflush(NULL);
|
fflush(NULL);
|
||||||
const size_t n_records = 10000;
|
const size_t n_records = 100;
|
||||||
std::string data;
|
std::string data;
|
||||||
std::vector<char> buf(1024);
|
std::vector<char> buf(1024);
|
||||||
// Generating data
|
// Generating data
|
||||||
@@ -352,7 +352,7 @@ namespace document_stream_tests {
|
|||||||
bool document_stream_utf8_test() {
|
bool document_stream_utf8_test() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
fflush(NULL);
|
fflush(NULL);
|
||||||
const size_t n_records = 10000;
|
const size_t n_records = 100;
|
||||||
std::string data;
|
std::string data;
|
||||||
std::vector<char> buf(1024);
|
std::vector<char> buf(1024);
|
||||||
// Generating data
|
// Generating data
|
||||||
@@ -388,7 +388,7 @@ namespace document_stream_tests {
|
|||||||
bool stress_data_race() {
|
bool stress_data_race() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
// Correct JSON.
|
// Correct JSON.
|
||||||
auto input = R"([1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;;
|
std::string input = R"([1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream));
|
||||||
@@ -404,7 +404,7 @@ namespace document_stream_tests {
|
|||||||
std::cout << "ENABLED" << std::endl;
|
std::cout << "ENABLED" << std::endl;
|
||||||
#endif
|
#endif
|
||||||
// Intentionally broken
|
// Intentionally broken
|
||||||
auto input = R"([1,23] [1,23] [1,23] [1,23 [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )"_padded;
|
std::string input = R"([1,23] [1,23] [1,23] [1,23 [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] [1,23] )";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document_stream stream;
|
ondemand::document_stream stream;
|
||||||
ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream));
|
ASSERT_SUCCESS(parser.iterate_many(input, 32).get(stream));
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ namespace error_tests {
|
|||||||
bool empty_document_error() {
|
bool empty_document_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
auto json = ""_padded;
|
std::string json = "";
|
||||||
ASSERT_ERROR( parser.iterate(json), EMPTY );
|
ASSERT_ERROR( parser.iterate(json), EMPTY );
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}
|
}
|
||||||
@@ -26,7 +26,7 @@ namespace error_tests {
|
|||||||
|
|
||||||
bool get_fail_then_succeed_bool() {
|
bool get_fail_then_succeed_bool() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "val" : true })"_padded;
|
std::string json = R"({ "val" : true })";
|
||||||
SUBTEST("simdjson_result<ondemand::value>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<ondemand::value>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::value> val = doc["val"];
|
simdjson_result<ondemand::value> val = doc["val"];
|
||||||
// Get everything that can fail in both forward and backwards order
|
// Get everything that can fail in both forward and backwards order
|
||||||
@@ -66,7 +66,7 @@ namespace error_tests {
|
|||||||
ASSERT_SUCCESS( val.get_bool() );
|
ASSERT_SUCCESS( val.get_bool() );
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}));
|
}));
|
||||||
json = R"(true)"_padded;
|
json = R"(true)";
|
||||||
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](simdjson_result<ondemand::document> val) {
|
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](simdjson_result<ondemand::document> val) {
|
||||||
// Get everything that can fail in both forward and backwards order
|
// Get everything that can fail in both forward and backwards order
|
||||||
ASSERT_EQUAL( val.is_null(), false );
|
ASSERT_EQUAL( val.is_null(), false );
|
||||||
@@ -110,7 +110,7 @@ namespace error_tests {
|
|||||||
|
|
||||||
bool get_fail_then_succeed_null() {
|
bool get_fail_then_succeed_null() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "val" : null })"_padded;
|
std::string json = R"({ "val" : null })";
|
||||||
SUBTEST("simdjson_result<ondemand::value>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<ondemand::value>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::value> val = doc["val"];
|
simdjson_result<ondemand::value> val = doc["val"];
|
||||||
// Get everything that can fail in both forward and backwards order
|
// Get everything that can fail in both forward and backwards order
|
||||||
@@ -150,7 +150,7 @@ namespace error_tests {
|
|||||||
ASSERT_EQUAL( val.is_null(), true );
|
ASSERT_EQUAL( val.is_null(), true );
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}));
|
}));
|
||||||
json = R"(null)"_padded;
|
json = R"(null)";
|
||||||
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](simdjson_result<ondemand::document> val) {
|
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](simdjson_result<ondemand::document> val) {
|
||||||
// Get everything that can fail in both forward and backwards order
|
// Get everything that can fail in both forward and backwards order
|
||||||
ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE );
|
ASSERT_ERROR( val.get_bool(), INCORRECT_TYPE );
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
using namespace simdjson;
|
using namespace simdjson;
|
||||||
|
|
||||||
namespace json_pointer_tests {
|
namespace json_pointer_tests {
|
||||||
const padded_string TEST_JSON = R"(
|
const std::string TEST_JSON = R"(
|
||||||
{
|
{
|
||||||
"/~01abc": [
|
"/~01abc": [
|
||||||
0,
|
0,
|
||||||
@@ -21,9 +21,9 @@ namespace json_pointer_tests {
|
|||||||
"": "empty ok",
|
"": "empty ok",
|
||||||
"arr": []
|
"arr": []
|
||||||
}
|
}
|
||||||
)"_padded;
|
)";
|
||||||
|
|
||||||
const padded_string TEST_RFC_JSON = R"(
|
const std::string TEST_RFC_JSON = R"(
|
||||||
{
|
{
|
||||||
"foo": ["bar", "baz"],
|
"foo": ["bar", "baz"],
|
||||||
"": 0,
|
"": 0,
|
||||||
@@ -36,7 +36,7 @@ namespace json_pointer_tests {
|
|||||||
" ": 7,
|
" ": 7,
|
||||||
"m~n": 8
|
"m~n": 8
|
||||||
}
|
}
|
||||||
)"_padded;
|
)";
|
||||||
|
|
||||||
bool run_success_test(const padded_string & json,std::string_view json_pointer,std::string expected) {
|
bool run_success_test(const padded_string & json,std::string_view json_pointer,std::string expected) {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
@@ -63,11 +63,11 @@ namespace json_pointer_tests {
|
|||||||
|
|
||||||
bool demo_test() {
|
bool demo_test() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document cars;
|
ondemand::document cars;
|
||||||
@@ -80,11 +80,11 @@ namespace json_pointer_tests {
|
|||||||
|
|
||||||
bool demo_relative_path() {
|
bool demo_relative_path() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document cars;
|
ondemand::document cars;
|
||||||
@@ -103,11 +103,11 @@ namespace json_pointer_tests {
|
|||||||
|
|
||||||
bool many_json_pointers() {
|
bool many_json_pointers() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document cars;
|
ondemand::document cars;
|
||||||
@@ -131,12 +131,12 @@ namespace json_pointer_tests {
|
|||||||
ondemand::value v;
|
ondemand::value v;
|
||||||
std::string_view val;
|
std::string_view val;
|
||||||
|
|
||||||
auto invalid_escape_key = R"( {"hello": [0,1,2,3], "te\est": "foo", "bool": true, "num":1234, "success":"yes"} )"_padded;
|
std::string invalid_escape_key = R"( {"hello": [0,1,2,3], "te\est": "foo", "bool": true, "num":1234, "success":"yes"} )";
|
||||||
auto invalid_escape_value = R"( {"hello": [0,1,2,3], "test": "fo\eo", "bool": true, "num":1234, "success":"yes"} )"_padded;
|
std::string invalid_escape_value = R"( {"hello": [0,1,2,3], "test": "fo\eo", "bool": true, "num":1234, "success":"yes"} )";
|
||||||
auto invalid_escape_value_at_jp = R"( {"hello": [0,1,2,3], "test": "foo", "bool": true, "num":1234, "success":"y\es"} )"_padded;
|
std::string invalid_escape_value_at_jp = R"( {"hello": [0,1,2,3], "test": "foo", "bool": true, "num":1234, "success":"y\es"} )";
|
||||||
auto unclosed_object = R"( {"test": "foo", "bool": true, "num":1234, "success":"yes" )"_padded;
|
std::string unclosed_object = R"( {"test": "foo", "bool": true, "num":1234, "success":"yes" )";
|
||||||
auto missing_bracket_before = R"( {"hello": [0,1,2,3, "test": "foo", "bool": true, "num":1234, "success":"yes"} )"_padded;
|
std::string missing_bracket_before = R"( {"hello": [0,1,2,3, "test": "foo", "bool": true, "num":1234, "success":"yes"} )";
|
||||||
auto missing_bracket_after = R"( {"test": "foo", "bool": true, "num":1234, "success":"yes", "hello":[0,1,2,3} )"_padded;
|
std::string missing_bracket_after = R"( {"test": "foo", "bool": true, "num":1234, "success":"yes", "hello":[0,1,2,3} )";
|
||||||
|
|
||||||
std::string json_pointer = "/success";
|
std::string json_pointer = "/success";
|
||||||
std::cout << "\t- invalid_escape_key" << std::endl;
|
std::cout << "\t- invalid_escape_key" << std::endl;
|
||||||
@@ -166,7 +166,7 @@ namespace json_pointer_tests {
|
|||||||
|
|
||||||
bool many_json_pointers_object_array() {
|
bool many_json_pointers_object_array() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto dogcatpotato = R"( { "dog" : [1,2,3], "cat" : [5, 6, 7], "potato" : [1234]})"_padded;
|
std::string dogcatpotato = R"( { "dog" : [1,2,3], "cat" : [5, 6, 7], "potato" : [1234]})";
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
@@ -182,7 +182,7 @@ namespace json_pointer_tests {
|
|||||||
}
|
}
|
||||||
bool many_json_pointers_object() {
|
bool many_json_pointers_object() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto cfoofoo2 = R"( { "c" :{ "foo": { "a": [ 10, 20, 30 ] }}, "d": { "foo2": { "a": [ 10, 20, 30 ] }} , "e": 120 })"_padded;
|
std::string cfoofoo2 = R"( { "c" :{ "foo": { "a": [ 10, 20, 30 ] }}, "d": { "foo2": { "a": [ 10, 20, 30 ] }} , "e": 120 })";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(cfoofoo2).get(doc));
|
ASSERT_SUCCESS(parser.iterate(cfoofoo2).get(doc));
|
||||||
@@ -199,7 +199,7 @@ namespace json_pointer_tests {
|
|||||||
}
|
}
|
||||||
bool many_json_pointers_array() {
|
bool many_json_pointers_array() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto cfoofoo2 = R"( [ 111, 2, 3, { "foo": { "a": [ 10, 20, 33 ] }}, { "foo2": { "a": [ 10, 20, 30 ] }}, 1001 ])"_padded;
|
std::string cfoofoo2 = R"( [ 111, 2, 3, { "foo": { "a": [ 10, 20, 33 ] }}, { "foo2": { "a": [ 10, 20, 30 ] }}, 1001 ])";
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(cfoofoo2).get(doc));
|
ASSERT_SUCCESS(parser.iterate(cfoofoo2).get(doc));
|
||||||
@@ -222,11 +222,11 @@ namespace json_pointer_tests {
|
|||||||
|
|
||||||
bool json_pointer_invalidation() {
|
bool json_pointer_invalidation() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto cars_json = R"( [
|
std::string cars_json = R"( [
|
||||||
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
|
||||||
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
|
||||||
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
|
||||||
] )"_padded;
|
] )";
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
ondemand::document cars;
|
ondemand::document cars;
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ namespace key_string_tests {
|
|||||||
bool parser_key_value() {
|
bool parser_key_value() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
const padded_string json = R"({ "1": "1", "2": "2", "3": "3", "abc": "abc", "\u0075": "\u0075" })"_padded;
|
const std::string json = R"({ "1": "1", "2": "2", "3": "3", "abc": "abc", "\u0075": "\u0075" })";
|
||||||
auto doc = parser.iterate(json);
|
auto doc = parser.iterate(json);
|
||||||
for(auto field : doc.get_object()) {
|
for(auto field : doc.get_object()) {
|
||||||
std::string_view keyv = field.unescaped_key();
|
std::string_view keyv = field.unescaped_key();
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ namespace misc_tests {
|
|||||||
bool issue1661a() {
|
bool issue1661a() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"":],"global-groups":[[]}})"_padded;
|
std::string docdata = R"({"":],"global-groups":[[]}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::value global_groups;
|
ondemand::value global_groups;
|
||||||
@@ -23,7 +23,7 @@ namespace misc_tests {
|
|||||||
bool issue1660() {
|
bool issue1660() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -54,7 +54,7 @@ namespace misc_tests {
|
|||||||
bool issue1660_with_bool() {
|
bool issue1660_with_bool() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -85,7 +85,7 @@ namespace misc_tests {
|
|||||||
bool issue1660_with_uint64() {
|
bool issue1660_with_uint64() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -116,7 +116,7 @@ namespace misc_tests {
|
|||||||
bool issue1660_with_int64() {
|
bool issue1660_with_int64() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -146,7 +146,7 @@ namespace misc_tests {
|
|||||||
bool issue1660_with_double() {
|
bool issue1660_with_double() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -177,7 +177,7 @@ namespace misc_tests {
|
|||||||
bool issue1660_with_null() {
|
bool issue1660_with_null() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -208,7 +208,7 @@ namespace misc_tests {
|
|||||||
bool issue1660_with_string() {
|
bool issue1660_with_string() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"globals":{"a":{"shadowable":[}}}})"_padded;
|
std::string docdata = R"({"globals":{"a":{"shadowable":[}}}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object globals;
|
ondemand::object globals;
|
||||||
@@ -238,7 +238,7 @@ namespace misc_tests {
|
|||||||
bool issue1661() {
|
bool issue1661() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string docdata = R"({"":],"global-groups":[[]}})"_padded;
|
std::string docdata = R"({"":],"global-groups":[[]}})";
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
ondemand::object global_groups;
|
ondemand::object global_groups;
|
||||||
@@ -252,7 +252,7 @@ namespace misc_tests {
|
|||||||
simdjson_warn_unused bool big_integer() {
|
simdjson_warn_unused bool big_integer() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"({"value":12321323213213213213213213213211223})"_padded;
|
std::string docdata = R"({"value":12321323213213213213213213213211223})";
|
||||||
simdjson::ondemand::document doc;
|
simdjson::ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
simdjson::ondemand::object o;
|
simdjson::ondemand::object o;
|
||||||
@@ -265,7 +265,7 @@ namespace misc_tests {
|
|||||||
simdjson_warn_unused bool big_integer_in_string() {
|
simdjson_warn_unused bool big_integer_in_string() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"({"value":"12321323213213213213213213213211223"})"_padded;
|
std::string docdata = R"({"value":"12321323213213213213213213213211223"})";
|
||||||
simdjson::ondemand::document doc;
|
simdjson::ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
simdjson::ondemand::object o;
|
simdjson::ondemand::object o;
|
||||||
@@ -275,17 +275,16 @@ namespace misc_tests {
|
|||||||
ASSERT_EQUAL(token, "\"12321323213213213213213213213211223\"");
|
ASSERT_EQUAL(token, "\"12321323213213213213213213213211223\"");
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
simdjson_warn_unused bool test_raw_json_token(string_view json, string_view expected_token, int expected_start_index = 0) {
|
simdjson_warn_unused bool test_raw_json_token(string json, string_view expected_token, int expected_start_index = 0) {
|
||||||
string title = "'";
|
string title = "'";
|
||||||
title.append(json.data(), json.length());
|
title.append(json.data(), json.length());
|
||||||
title += "'";
|
title += "'";
|
||||||
padded_string json_padded = json;
|
SUBTEST(title, test_ondemand_doc(json, [&](auto doc) {
|
||||||
SUBTEST(title, test_ondemand_doc(json_padded, [&](auto doc) {
|
|
||||||
string_view token;
|
string_view token;
|
||||||
ASSERT_SUCCESS( doc.raw_json_token().get(token) );
|
ASSERT_SUCCESS( doc.raw_json_token().get(token) );
|
||||||
ASSERT_EQUAL( token, expected_token );
|
ASSERT_EQUAL( token, expected_token );
|
||||||
// Validate the text is inside the original buffer
|
// Validate the text is inside the original buffer
|
||||||
ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_padded.data()[expected_start_index]));
|
ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json.data()[expected_start_index]));
|
||||||
return true;
|
return true;
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@@ -293,22 +292,22 @@ namespace misc_tests {
|
|||||||
auto json_in_hash = string(R"({"a":)");
|
auto json_in_hash = string(R"({"a":)");
|
||||||
json_in_hash.append(json.data(), json.length());
|
json_in_hash.append(json.data(), json.length());
|
||||||
json_in_hash += "}";
|
json_in_hash += "}";
|
||||||
json_padded = json_in_hash;
|
|
||||||
title = "'";
|
title = "'";
|
||||||
title.append(json_in_hash.data(), json_in_hash.length());
|
title.append(json_in_hash.data(), json_in_hash.length());
|
||||||
title += "'";
|
title += "'";
|
||||||
SUBTEST(title, test_ondemand_doc(json_padded, [&](auto doc) {
|
SUBTEST(title, test_ondemand_doc(json_in_hash, [&](auto doc) {
|
||||||
string_view token;
|
string_view token;
|
||||||
ASSERT_SUCCESS( doc["a"].raw_json_token().get(token) );
|
ASSERT_SUCCESS( doc["a"].raw_json_token().get(token) );
|
||||||
ASSERT_EQUAL( token, expected_token );
|
ASSERT_EQUAL( token, expected_token );
|
||||||
// Validate the text is inside the original buffer
|
// Validate the text is inside the original buffer
|
||||||
// Adjust for the {"a":
|
// Adjust for the {"a":
|
||||||
ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_padded.data()[5+expected_start_index]));
|
ASSERT_EQUAL( reinterpret_cast<const void*>(token.data()), reinterpret_cast<const void*>(&json_in_hash.data()[5+expected_start_index]));
|
||||||
return true;
|
return true;
|
||||||
}));
|
}));
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
//bool raw_json_token() { return true; }
|
||||||
|
|
||||||
bool raw_json_token() {
|
bool raw_json_token() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ namespace number_tests {
|
|||||||
std::cout << __func__ << std::endl;
|
std::cout << __func__ << std::endl;
|
||||||
|
|
||||||
// converts the double "expected" to a padded string
|
// converts the double "expected" to a padded string
|
||||||
auto format_into_padded=[](const double expected) -> padded_string
|
auto format_into_padded=[](const double expected) -> std::string
|
||||||
{
|
{
|
||||||
std::vector<char> buf(1024);
|
std::vector<char> buf(1024);
|
||||||
const auto n = std::snprintf(buf.data(),
|
const auto n = std::snprintf(buf.data(),
|
||||||
@@ -36,7 +36,7 @@ namespace number_tests {
|
|||||||
expected);
|
expected);
|
||||||
const auto nz=static_cast<size_t>(n);
|
const auto nz=static_cast<size_t>(n);
|
||||||
if (n<0 || nz >= buf.size()) { std::abort(); }
|
if (n<0 || nz >= buf.size()) { std::abort(); }
|
||||||
return padded_string(buf.data(), nz);
|
return std::string(buf.data(), nz);
|
||||||
};
|
};
|
||||||
|
|
||||||
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
|
for (int i = -1075; i < 1024; ++i) {// large negative values should be zero.
|
||||||
@@ -146,8 +146,8 @@ namespace number_tests {
|
|||||||
if (n >= buf.size()) { std::abort(); }
|
if (n >= buf.size()) { std::abort(); }
|
||||||
std::fflush(nullptr);
|
std::fflush(nullptr);
|
||||||
const double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i));
|
const double expected = ((i >= -307) ? testing_power_of_ten[i + 307]: std::pow(10, i));
|
||||||
|
std::string str(buf.data(), n);
|
||||||
if(!test_ondemand<double>(padded_string(buf.data(), n), [&](double actual) {
|
if(!test_ondemand<double>(str, [&](double actual) {
|
||||||
if(actual!=expected) {
|
if(actual!=expected) {
|
||||||
std::cerr << "JSON '" << buf.data() << " parsed to ";
|
std::cerr << "JSON '" << buf.data() << " parsed to ";
|
||||||
std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
|
std::fprintf( stderr," %18.18g instead of %18.18g\n", actual, expected); // formatting numbers is easier with printf
|
||||||
|
|||||||
@@ -88,10 +88,10 @@ namespace object_error_tests {
|
|||||||
}
|
}
|
||||||
bool object_lookup_unclosed_error() {
|
bool object_lookup_unclosed_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
#if __SIMDJSON_CHECK_EOF
|
#if SIMDJSON_CHECK_EOF
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
#else
|
#else
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_success(doc["a"]));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
#endif
|
#endif
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
@@ -119,13 +119,11 @@ namespace object_error_tests {
|
|||||||
bool object_lookup_miss_unclosed_error() {
|
bool object_lookup_miss_unclosed_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1, )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1, )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
// TODO These next two pass the user a value that may run past the end of the buffer if they aren't careful.
|
|
||||||
// In particular, if the padding is decorated with the wrong values, we could cause overrun!
|
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1 )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a": 1 )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["b"], INCOMPLETE_ARRAY_OR_OBJECT));
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}
|
}
|
||||||
bool object_lookup_miss_next_error() {
|
bool object_lookup_miss_next_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
@@ -139,7 +137,7 @@ namespace object_error_tests {
|
|||||||
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
#ifdef SIMDJSON_DEVELOPMENT_CHECKS
|
||||||
bool out_of_order_object_iteration_error() {
|
bool out_of_order_object_iteration_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 1, "y": 2 } ])"_padded;
|
std::string json = R"([ { "x": 1, "y": 2 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
for (auto element : doc) {
|
for (auto element : doc) {
|
||||||
auto obj = element.get_object();
|
auto obj = element.get_object();
|
||||||
@@ -162,7 +160,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_top_level_object_iteration_error() {
|
bool out_of_order_top_level_object_iteration_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "x": 1, "y": 2 })"_padded;
|
std::string json = R"({ "x": 1, "y": 2 })";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
auto obj = doc.get_object();
|
auto obj = doc.get_object();
|
||||||
for (auto field : obj) { ASSERT_SUCCESS(field); }
|
for (auto field : obj) { ASSERT_SUCCESS(field); }
|
||||||
@@ -181,7 +179,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_object_index_child_error() {
|
bool out_of_order_object_index_child_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 1, "y": 2 } ])"_padded;
|
std::string json = R"([ { "x": 1, "y": 2 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::object> obj;
|
simdjson_result<ondemand::object> obj;
|
||||||
for (auto element : doc) {
|
for (auto element : doc) {
|
||||||
@@ -223,7 +221,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_object_index_sibling_error() {
|
bool out_of_order_object_index_sibling_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])"_padded;
|
std::string json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::object> last_obj;
|
simdjson_result<ondemand::object> last_obj;
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
@@ -302,7 +300,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_object_find_field_child_error() {
|
bool out_of_order_object_find_field_child_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 1, "y": 2 } ])"_padded;
|
std::string json = R"([ { "x": 1, "y": 2 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::object> obj;
|
simdjson_result<ondemand::object> obj;
|
||||||
for (auto element : doc) {
|
for (auto element : doc) {
|
||||||
@@ -344,7 +342,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_object_find_field_sibling_error() {
|
bool out_of_order_object_find_field_sibling_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])"_padded;
|
std::string json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::object> last_obj;
|
simdjson_result<ondemand::object> last_obj;
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
@@ -423,7 +421,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_object_find_field_unordered_child_error() {
|
bool out_of_order_object_find_field_unordered_child_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 1, "y": 2 } ])"_padded;
|
std::string json = R"([ { "x": 1, "y": 2 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::object> obj;
|
simdjson_result<ondemand::object> obj;
|
||||||
for (auto element : doc) {
|
for (auto element : doc) {
|
||||||
@@ -465,7 +463,7 @@ namespace object_error_tests {
|
|||||||
|
|
||||||
bool out_of_order_object_find_field_unordered_sibling_error() {
|
bool out_of_order_object_find_field_unordered_sibling_error() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])"_padded;
|
std::string json = R"([ { "x": 0, "y": 2 }, { "x": 1, "y": 4 } ])";
|
||||||
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
SUBTEST("simdjson_result<object>", test_ondemand_doc(json, [&](auto doc) {
|
||||||
simdjson_result<ondemand::object> last_obj;
|
simdjson_result<ondemand::object> last_obj;
|
||||||
uint64_t i = 0;
|
uint64_t i = 0;
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool object_find_field_unordered() {
|
bool object_find_field_unordered() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
ASSERT_SUCCESS( doc_result.get(object) );
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
@@ -39,7 +39,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool document_object_find_field_unordered() {
|
bool document_object_find_field_unordered() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::document", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::document", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS( std::move(doc_result).get(doc) );
|
ASSERT_SUCCESS( std::move(doc_result).get(doc) );
|
||||||
@@ -65,7 +65,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool value_object_find_field_unordered() {
|
bool value_object_find_field_unordered() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })"_padded;
|
std::string json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })";
|
||||||
SUBTEST("ondemand::value", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::value", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::value object;
|
ondemand::value object;
|
||||||
ASSERT_SUCCESS( doc_result.find_field_unordered("outer").get(object) );
|
ASSERT_SUCCESS( doc_result.find_field_unordered("outer").get(object) );
|
||||||
@@ -92,7 +92,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool object_find_field() {
|
bool object_find_field() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
ASSERT_SUCCESS( doc_result.get(object) );
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
@@ -122,7 +122,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool document_object_find_field() {
|
bool document_object_find_field() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::document", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::document", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS( std::move(doc_result).get(doc) );
|
ASSERT_SUCCESS( std::move(doc_result).get(doc) );
|
||||||
@@ -148,7 +148,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool value_object_find_field() {
|
bool value_object_find_field() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })"_padded;
|
std::string json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })";
|
||||||
SUBTEST("ondemand::value", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::value", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::value object;
|
ondemand::value object;
|
||||||
ASSERT_SUCCESS( doc_result.find_field("outer").get(object) );
|
ASSERT_SUCCESS( doc_result.find_field("outer").get(object) );
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool object_index() {
|
bool object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
ASSERT_SUCCESS( doc_result.get(object) );
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
@@ -39,7 +39,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool document_object_index() {
|
bool document_object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::document", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::document", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::document doc;
|
ondemand::document doc;
|
||||||
ASSERT_SUCCESS( std::move(doc_result).get(doc) );
|
ASSERT_SUCCESS( std::move(doc_result).get(doc) );
|
||||||
@@ -65,7 +65,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool value_object_index() {
|
bool value_object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })"_padded;
|
std::string json = R"({ "outer": { "a": 1, "b": 2, "c/d": 3 } })";
|
||||||
SUBTEST("ondemand::value", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::value", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::value object;
|
ondemand::value object;
|
||||||
ASSERT_SUCCESS( doc_result["outer"].get(object) );
|
ASSERT_SUCCESS( doc_result["outer"].get(object) );
|
||||||
@@ -92,7 +92,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool document_nested_object_index() {
|
bool document_nested_object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded;
|
std::string json = R"({ "x": { "y": { "z": 2 } } }})";
|
||||||
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ASSERT_EQUAL( doc_result["x"]["y"]["z"].get_uint64().value_unsafe(), 2 );
|
ASSERT_EQUAL( doc_result["x"]["y"]["z"].get_uint64().value_unsafe(), 2 );
|
||||||
return true;
|
return true;
|
||||||
@@ -108,7 +108,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool nested_object_index() {
|
bool nested_object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded;
|
std::string json = R"({ "x": { "y": { "z": 2 } } }})";
|
||||||
SUBTEST("simdjson_result<ondemand::object>", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("simdjson_result<ondemand::object>", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
simdjson_result<ondemand::object> object = doc_result.get_object();
|
simdjson_result<ondemand::object> object = doc_result.get_object();
|
||||||
ASSERT_EQUAL( object["x"]["y"]["z"].get_uint64().value_unsafe(), 2 );
|
ASSERT_EQUAL( object["x"]["y"]["z"].get_uint64().value_unsafe(), 2 );
|
||||||
@@ -125,7 +125,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool value_nested_object_index() {
|
bool value_nested_object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded;
|
std::string json = R"({ "x": { "y": { "z": 2 } } }})";
|
||||||
SUBTEST("simdjson_result<ondemand::value>", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("simdjson_result<ondemand::value>", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
simdjson_result<ondemand::value> x = doc_result["x"];
|
simdjson_result<ondemand::value> x = doc_result["x"];
|
||||||
ASSERT_EQUAL( x["y"]["z"].get_uint64().value_unsafe(), 2 );
|
ASSERT_EQUAL( x["y"]["z"].get_uint64().value_unsafe(), 2 );
|
||||||
@@ -142,7 +142,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool object_index_partial_children() {
|
bool object_index_partial_children() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"(
|
std::string json = R"(
|
||||||
{
|
{
|
||||||
"scalar_ignore": 0,
|
"scalar_ignore": 0,
|
||||||
"empty_array_ignore": [],
|
"empty_array_ignore": [],
|
||||||
@@ -156,7 +156,7 @@ namespace object_tests {
|
|||||||
"quadruple_nested_break": { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 },
|
"quadruple_nested_break": { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 },
|
||||||
"actual_value": 10
|
"actual_value": 10
|
||||||
}
|
}
|
||||||
)"_padded;
|
)";
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
ASSERT_SUCCESS( doc_result.get(object) );
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
@@ -385,7 +385,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool object_index_exception() {
|
bool object_index_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c/d": 3})"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c/d": 3})";
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object = doc_result;
|
ondemand::object object = doc_result;
|
||||||
|
|
||||||
@@ -399,7 +399,7 @@ namespace object_tests {
|
|||||||
}
|
}
|
||||||
bool nested_object_index_exception() {
|
bool nested_object_index_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "x": { "y": { "z": 2 } } }})"_padded;
|
std::string json = R"({ "x": { "y": { "z": 2 } } }})";
|
||||||
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("simdjson_result<ondemand::document>", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ASSERT_EQUAL( uint64_t(doc_result["x"]["y"]["z"]), 2 );
|
ASSERT_EQUAL( uint64_t(doc_result["x"]["y"]["z"]), 2 );
|
||||||
return true;
|
return true;
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ namespace object_tests {
|
|||||||
bool no_missing_keys() {
|
bool no_missing_keys() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"([{"a":"a"},{}])"_padded;
|
std::string docdata = R"([{"a":"a"},{}])";
|
||||||
simdjson::ondemand::document doc;
|
simdjson::ondemand::document doc;
|
||||||
auto error = parser.iterate(docdata).get(doc);
|
auto error = parser.iterate(docdata).get(doc);
|
||||||
if(error != simdjson::SUCCESS) { return false; }
|
if(error != simdjson::SUCCESS) { return false; }
|
||||||
@@ -33,7 +33,7 @@ namespace object_tests {
|
|||||||
bool missing_key_continue() {
|
bool missing_key_continue() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"({"a":0, "b":1, "c":2})"_padded;
|
std::string docdata = R"({"a":0, "b":1, "c":2})";
|
||||||
simdjson::ondemand::document doc;
|
simdjson::ondemand::document doc;
|
||||||
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
ASSERT_SUCCESS(parser.iterate(docdata).get(doc));
|
||||||
int64_t num;
|
int64_t num;
|
||||||
@@ -90,9 +90,12 @@ namespace object_tests {
|
|||||||
bool missing_keys() {
|
bool missing_keys() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = R"([{"a":"a"},{}])"_padded;
|
// We want to stress memory overflows.
|
||||||
|
const char* staticdoc = R"([{"a":"a"},{}])";
|
||||||
|
std::unique_ptr<char[]> docdata(new char[strlen(staticdoc)]);
|
||||||
|
std::memcpy(docdata.get(), staticdoc, strlen(staticdoc));
|
||||||
simdjson::ondemand::document doc;
|
simdjson::ondemand::document doc;
|
||||||
auto error = parser.iterate(docdata).get(doc);
|
auto error = parser.iterate(docdata.get(), strlen(staticdoc)).get(doc);
|
||||||
if(error != simdjson::SUCCESS) { return false; }
|
if(error != simdjson::SUCCESS) { return false; }
|
||||||
simdjson::ondemand::array a;
|
simdjson::ondemand::array a;
|
||||||
error = doc.get_array().get(a);
|
error = doc.get_array().get(a);
|
||||||
@@ -110,9 +113,12 @@ namespace object_tests {
|
|||||||
bool missing_keys_for_empty_top_level_object() {
|
bool missing_keys_for_empty_top_level_object() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
simdjson::ondemand::parser parser;
|
simdjson::ondemand::parser parser;
|
||||||
simdjson::padded_string docdata = "{}"_padded;
|
std::unique_ptr<char[]> docdata(new char[2]);
|
||||||
|
// We want to stress memory overflows.
|
||||||
|
const char* staticdoc = "{}";
|
||||||
|
std::memcpy(docdata.get(), staticdoc, 2);
|
||||||
simdjson::ondemand::document doc;
|
simdjson::ondemand::document doc;
|
||||||
auto error = parser.iterate(docdata).get(doc);
|
auto error = parser.iterate(docdata.get(), 2).get(doc);
|
||||||
if(error != simdjson::SUCCESS) { return false; }
|
if(error != simdjson::SUCCESS) { return false; }
|
||||||
error = doc.find_field_unordered("keynotfound").error();
|
error = doc.find_field_unordered("keynotfound").error();
|
||||||
if(error != simdjson::NO_SUCH_FIELD) {
|
if(error != simdjson::NO_SUCH_FIELD) {
|
||||||
@@ -139,7 +145,7 @@ namespace object_tests {
|
|||||||
bool broken_issue_1521() {
|
bool broken_issue_1521() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string json = R"({"type":"root","nodes":[{"type":"child","nodes":[]},{"type":"child","name":"child-name","nodes":[]}]})"_padded;
|
std::string json = R"({"type":"root","nodes":[{"type":"child","nodes":[]},{"type":"child","name":"child-name","nodes":[]}]})";
|
||||||
ondemand::document file_tree = parser.iterate(json);
|
ondemand::document file_tree = parser.iterate(json);
|
||||||
try {
|
try {
|
||||||
broken_descend(file_tree);
|
broken_descend(file_tree);
|
||||||
@@ -154,7 +160,7 @@ namespace object_tests {
|
|||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
// We omit the ',"nodes":[]'
|
// We omit the ',"nodes":[]'
|
||||||
padded_string json = R"({"type":"root","nodes":[{"type":"child"},{"type":"child","name":"child-name","nodes":[]}]})"_padded;
|
std::string json = R"({"type":"root","nodes":[{"type":"child"},{"type":"child","name":"child-name","nodes":[]}]})";
|
||||||
ondemand::document file_tree = parser.iterate(json);
|
ondemand::document file_tree = parser.iterate(json);
|
||||||
try {
|
try {
|
||||||
broken_descend(file_tree);
|
broken_descend(file_tree);
|
||||||
@@ -181,7 +187,7 @@ namespace object_tests {
|
|||||||
bool issue_1521() {
|
bool issue_1521() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
padded_string json = R"({"type":"root","nodes":[{"type":"child","nodes":[]},{"type":"child","name":"child-name","nodes":[]}]})"_padded;
|
std::string json = R"({"type":"root","nodes":[{"type":"child","nodes":[]},{"type":"child","name":"child-name","nodes":[]}]})";
|
||||||
ondemand::document file_tree = parser.iterate(json);
|
ondemand::document file_tree = parser.iterate(json);
|
||||||
try {
|
try {
|
||||||
descend(file_tree);
|
descend(file_tree);
|
||||||
@@ -195,7 +201,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool iterate_object() {
|
bool iterate_object() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c": 3 })"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c": 3 })";
|
||||||
const char* expected_key[] = { "a", "b", "c" };
|
const char* expected_key[] = { "a", "b", "c" };
|
||||||
const uint64_t expected_value[] = { 1, 2, 3 };
|
const uint64_t expected_value[] = { 1, 2, 3 };
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
@@ -254,7 +260,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool iterate_object_partial_children() {
|
bool iterate_object_partial_children() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"(
|
std::string json = R"(
|
||||||
{
|
{
|
||||||
"scalar_ignore": 0,
|
"scalar_ignore": 0,
|
||||||
"empty_array_ignore": [],
|
"empty_array_ignore": [],
|
||||||
@@ -268,7 +274,7 @@ namespace object_tests {
|
|||||||
"quadruple_nested_break": { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 },
|
"quadruple_nested_break": { "a": [ { "b": [ 9, 99 ], "c": 999 }, 9999 ], "d": 99999 },
|
||||||
"actual_value": 10
|
"actual_value": 10
|
||||||
}
|
}
|
||||||
)"_padded;
|
)";
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
ASSERT_SUCCESS( doc_result.get(object) );
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
@@ -408,7 +414,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool iterate_empty_object() {
|
bool iterate_empty_object() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({})"_padded;
|
std::string json = R"({})";
|
||||||
|
|
||||||
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
@@ -430,7 +436,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool value_search_unescaped_key() {
|
bool value_search_unescaped_key() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({"k\u0065y": 1})"_padded;
|
std::string json = R"({"k\u0065y": 1})";
|
||||||
SUBTEST("ondemand::unescapedkey", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::unescapedkey", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
bool got_key = false;
|
bool got_key = false;
|
||||||
@@ -468,7 +474,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool issue_1480() {
|
bool issue_1480() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "name" : "something", "version": "0.13.2", "version_major": 0})"_padded;
|
std::string json = R"({ "name" : "something", "version": "0.13.2", "version_major": 0})";
|
||||||
|
|
||||||
SUBTEST("ondemand::issue_1480::object", test_ondemand_doc(json, [&](auto doc_result) {
|
SUBTEST("ondemand::issue_1480::object", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
ondemand::object object;
|
ondemand::object object;
|
||||||
@@ -545,7 +551,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool iterate_object_exception() {
|
bool iterate_object_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({ "a": 1, "b": 2, "c": 3 })"_padded;
|
std::string json = R"({ "a": 1, "b": 2, "c": 3 })";
|
||||||
const char* expected_key[] = { "a", "b", "c" };
|
const char* expected_key[] = { "a", "b", "c" };
|
||||||
const uint64_t expected_value[] = { 1, 2, 3 };
|
const uint64_t expected_value[] = { 1, 2, 3 };
|
||||||
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
@@ -563,7 +569,7 @@ namespace object_tests {
|
|||||||
|
|
||||||
bool iterate_empty_object_exception() {
|
bool iterate_empty_object_exception() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
auto json = R"({})"_padded;
|
std::string json = R"({})";
|
||||||
|
|
||||||
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
ASSERT_TRUE(test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
for (simdjson_unused ondemand::field field : doc_result.get_object()) {
|
for (simdjson_unused ondemand::field field : doc_result.get_object()) {
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ namespace ordering_tests {
|
|||||||
|
|
||||||
#if SIMDJSON_EXCEPTIONS
|
#if SIMDJSON_EXCEPTIONS
|
||||||
|
|
||||||
auto json = "{\"coordinates\":[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}"_padded;
|
std::string json = "{\"coordinates\":[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}";
|
||||||
|
|
||||||
bool in_order_object_index() {
|
bool in_order_object_index() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
@@ -133,7 +133,7 @@ namespace ordering_tests {
|
|||||||
bool use_object_multiple_times_out_of_order() {
|
bool use_object_multiple_times_out_of_order() {
|
||||||
TEST_START();
|
TEST_START();
|
||||||
ondemand::parser parser{};
|
ondemand::parser parser{};
|
||||||
auto json2 = "{\"coordinates\":{\"x\":1.1,\"y\":2.2,\"z\":3.3}}"_padded;
|
std::string json2 = "{\"coordinates\":{\"x\":1.1,\"y\":2.2,\"z\":3.3}}";
|
||||||
auto doc = parser.iterate(json2);
|
auto doc = parser.iterate(json2);
|
||||||
auto x = doc["coordinates"]["x"];
|
auto x = doc["coordinates"]["x"];
|
||||||
auto y = doc["coordinates"]["y"];
|
auto y = doc["coordinates"]["y"];
|
||||||
|
|||||||
@@ -6,9 +6,9 @@ using namespace simdjson;
|
|||||||
namespace parse_api_tests {
|
namespace parse_api_tests {
|
||||||
using namespace std;
|
using namespace std;
|
||||||
|
|
||||||
const padded_string BASIC_JSON = "[1,2,3]"_padded;
|
const std::string BASIC_JSON = "[1,2,3]";
|
||||||
const padded_string BASIC_NDJSON = "[1,2,3]\n[4,5,6]"_padded;
|
const std::string BASIC_NDJSON = "[1,2,3]\n[4,5,6]";
|
||||||
const padded_string EMPTY_NDJSON = ""_padded;
|
const std::string EMPTY_NDJSON = "";
|
||||||
|
|
||||||
|
|
||||||
bool parser_iterate_empty() {
|
bool parser_iterate_empty() {
|
||||||
@@ -109,13 +109,6 @@ namespace parse_api_tests {
|
|||||||
ASSERT_SUCCESS( doc.get_double() );
|
ASSERT_SUCCESS( doc.get_double() );
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
|
||||||
cout << "- padded_string_view(string_view(char*))" << endl;
|
|
||||||
padded_string_view json(json_str, sizeof(json_str));
|
|
||||||
auto doc = parser.iterate(json);
|
|
||||||
ASSERT_SUCCESS( doc.get_double() );
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,30 +121,6 @@ namespace parse_api_tests {
|
|||||||
ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 31);
|
ASSERT_EQUAL(padded_string_view(json_str, strlen(json_str), sizeof(json_str)).padding(), 31);
|
||||||
ASSERT_EQUAL(SIMDJSON_PADDING, 32);
|
ASSERT_EQUAL(SIMDJSON_PADDING, 32);
|
||||||
|
|
||||||
{
|
|
||||||
cout << "- char*, 31 padding" << endl;
|
|
||||||
ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), sizeof(json_str)), INSUFFICIENT_PADDING );
|
|
||||||
cout << "- char*, 0 padding" << endl;
|
|
||||||
ASSERT_ERROR( parser.iterate(json_str, strlen(json_str), strlen(json_str)), INSUFFICIENT_PADDING );
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
std::string_view json(json_str);
|
|
||||||
cout << "- string_view, 31 padding" << endl;
|
|
||||||
ASSERT_ERROR( parser.iterate(json, sizeof(json_str)), INSUFFICIENT_PADDING );
|
|
||||||
cout << "- string_view, 0 padding" << endl;
|
|
||||||
ASSERT_ERROR( parser.iterate(json, strlen(json_str)), INSUFFICIENT_PADDING );
|
|
||||||
}
|
|
||||||
|
|
||||||
{
|
|
||||||
std::string json = "12";
|
|
||||||
json.shrink_to_fit();
|
|
||||||
cout << "- string, 0 padding" << endl;
|
|
||||||
ASSERT_ERROR( parser.iterate(json), INSUFFICIENT_PADDING );
|
|
||||||
// It's actually kind of hard to allocate "just enough" capacity, since the string tends
|
|
||||||
// to grow more than you tell it to.
|
|
||||||
}
|
|
||||||
|
|
||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,9 +139,9 @@ namespace parse_api_tests {
|
|||||||
// A document spans about 40 bytes. Nevertheless, some users
|
// A document spans about 40 bytes. Nevertheless, some users
|
||||||
// would rather reuse them.
|
// would rather reuse them.
|
||||||
std::cout << sizeof(doc) << std::endl;
|
std::cout << sizeof(doc) << std::endl;
|
||||||
auto json = R"({"key": "value"})"_padded;
|
std::string json = R"({"key": "value"})";
|
||||||
auto jsonbad = R"({"key": "value")"_padded; // deliberaty broken
|
std::string jsonbad = R"({"key": "value")"; // deliberaty broken
|
||||||
auto jsonunclosedstring = "{\"coordinates:[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}"_padded;
|
std::string jsonunclosedstring = "{\"coordinates:[{\"x\":1.1,\"y\":2.2,\"z\":3.3}]}";
|
||||||
std::string_view output;
|
std::string_view output;
|
||||||
|
|
||||||
ondemand::parser parser;
|
ondemand::parser parser;
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user