From b79261eebcd7b9a784f1e2d17de904841713f80c Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sun, 25 Jul 2021 10:36:22 -0400 Subject: [PATCH] This cleans a bit the current code, especially with respect to EOF guards. (#1669) * Upgrading the GitHub Actions. * Upgrading appveyor * Upgrading circle ci. * Cleaning. --- .appveyor.yml | 10 +-- .circleci/config.yml | 28 +++------ .github/workflows/alpine.yml | 8 +-- .github/workflows/fix-trailing-whitespace.yml | 5 +- .github/workflows/mingw-ci.yml | 11 +--- .github/workflows/mingw64-ci.yml | 9 +-- .github/workflows/msys2-clang.yml | 9 +-- .github/workflows/msys2.yml | 8 +-- .github/workflows/ubuntu18-threadsani.yml | 9 +-- .github/workflows/ubuntu18.yml | 8 +-- .github/workflows/ubuntu20-noexcept.yml | 8 +-- .github/workflows/ubuntu20-nothread.yml | 10 +-- .github/workflows/ubuntu20-sani.yml | 23 +++++++ .github/workflows/ubuntu20-threadsani.yml | 8 +-- .github/workflows/ubuntu20.yml | 8 +-- .github/workflows/vs15-ci.yml | 8 +-- .github/workflows/vs16-ci.yml | 8 +-- .github/workflows/vs16-clang-ci.yml | 10 +-- .github/workflows/vs16-ninja-ci.yml | 8 +-- .github/workflows/vs16-noexcept-ci.yml | 8 +-- cmake/developer-options.cmake | 6 ++ doc/basics.md | 2 +- include/simdjson/arm64/numberparsing.h | 2 +- include/simdjson/common_defs.h | 13 ++-- include/simdjson/fallback/numberparsing.h | 3 +- include/simdjson/generic/numberparsing.h | 10 +-- include/simdjson/generic/ondemand/array-inl.h | 5 +- .../simdjson/generic/ondemand/document-inl.h | 6 +- .../generic/ondemand/json_iterator-inl.h | 37 ++++++----- .../simdjson/generic/ondemand/json_iterator.h | 13 +++- include/simdjson/generic/ondemand/parser.h | 63 ++++++++++++++++++- .../generic/ondemand/value_iterator-inl.h | 7 ++- include/simdjson/haswell/numberparsing.h | 2 +- include/simdjson/ppc64/numberparsing.h | 2 +- include/simdjson/westmere/numberparsing.h | 2 +- singleheader/simdjson.h | 20 +++--- tests/dom/basictests.cpp | 19 +++++- tests/dom/document_stream_tests.cpp | 4 +- tests/dom/document_tests.cpp | 2 +- tests/ondemand/ondemand_array_error_tests.cpp | 6 ++ .../ondemand_document_stream_tests.cpp | 4 +- .../ondemand/ondemand_object_error_tests.cpp | 4 +- 42 files changed, 230 insertions(+), 206 deletions(-) create mode 100644 .github/workflows/ubuntu20-sani.yml diff --git a/.appveyor.yml b/.appveyor.yml index 2d12fe3ae..cdf8f00d4 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -15,22 +15,22 @@ environment: - job_name: VS2019 CMAKE_ARGS: -A %Platform% - job_name: VS2019ARM - CMAKE_ARGS: -A ARM64 -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM? + CMAKE_ARGS: -A ARM64 -DSIMDJSON_DEVELOPER_MODE=ON -DCMAKE_CROSSCOMPILING=1 -D SIMDJSON_GOOGLE_BENCHMARKS=OFF # Does Google Benchmark builds under VS ARM? - job_name: VS2017 (Static, No Threads) image: Visual Studio 2017 - CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32) platform: Win32 - CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. + CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON # This should be the default. Testing anyway. CTEST_ARGS: -LE explicitonly - job_name: VS2019 (Win32, No Exceptions) platform: Win32 - CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF + CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=ON -DSIMDJSON_ENABLE_THREADS=ON -DSIMDJSON_EXCEPTIONS=OFF CTEST_ARGS: -LE explicitonly - job_name: VS2015 image: Visual Studio 2015 - CMAKE_ARGS: -A %Platform% -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF + CMAKE_ARGS: -A %Platform% -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_ENABLE_THREADS=OFF CTEST_ARGS: -LE explicitonly build_script: diff --git a/.circleci/config.yml b/.circleci/config.yml index c96ee86ee..9b1f48ed7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 -# We constantly run out of memory so please do not use parallelism (-j, -j4). +# We constantly run out of memory so please do not use parallelism (-j, -j4). # Reusable image / compiler definitions executors: @@ -48,7 +48,7 @@ executors: CXX: clang++-9 CC: clang-9 CMAKE_BUILD_FLAGS: - CTEST_FLAGS: --output-on-failure + CTEST_FLAGS: --output-on-failure clang6: docker: @@ -88,13 +88,13 @@ commands: steps: - cmake_prep - dependency_restore - - run: cmake $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build . + - run: cmake -DSIMDJSON_DEVELOPER_MODE=ON $CMAKE_FLAGS -DCMAKE_INSTALL_PREFIX:PATH=destination -B build . - dependency_cache # dependencies are produced in the configure step cmake_build: steps: - cmake_build_cache - - run: cmake --build build + - run: cmake --build build cmake_test: steps: @@ -130,7 +130,7 @@ commands: - cmake_build_cache - run: | cmake -DSIMDJSON_ENABLE_DOM_CHECKPERF=ON --build build --target checkperf && - cd build && + cd build && ctest --output-on-failure -R checkperf # we not only want cmake to build and run tests, but we want also a successful installation from which we can build, link and run programs @@ -191,12 +191,12 @@ jobs: sanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] threadsanitize-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build @@ -206,15 +206,7 @@ jobs: threadsanitize-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } - steps: [ cmake_test ] - nocheckeof-clang10: - description: Validate that when __SIMDJSON_CHECK_EOF=0, everything still succeeds - environment: - CXXFLAGS: -D__SIMDJSON_CHECK_EOF=0 - CMAKE_BUILD_FLAGS: --target ondemand_tests - CTEST_FLAGS: --output-on-failure -R ondemand_ - executor: clang10 + environment: { CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE_THREADS=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] # dynamic dynamic-gcc10: @@ -270,12 +262,12 @@ jobs: sanitize-haswell-gcc10: description: Build and run tests on GCC 10 and AVX 2 with a cmake sanitize build executor: gcc10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] sanitize-haswell-clang10: description: Build and run tests on clang 10 and AVX 2 with a cmake sanitize build executor: clang10 - environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } + environment: { CXXFLAGS: -march=haswell, CMAKE_FLAGS: -DBUILD_SHARED_LIBS=ON -DSIMDJSON_NO_FORCE_INLINING=ON -DSIMDJSON_SANITIZE=ON, CTEST_FLAGS: --output-on-failure -LE explicitonly } steps: [ cmake_test ] workflows: diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml index 799f483ad..24d65a126 100644 --- a/.github/workflows/alpine.yml +++ b/.github/workflows/alpine.yml @@ -1,12 +1,6 @@ name: Alpine Linux -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ubuntu-build: diff --git a/.github/workflows/fix-trailing-whitespace.yml b/.github/workflows/fix-trailing-whitespace.yml index 63706869d..890ca78c2 100644 --- a/.github/workflows/fix-trailing-whitespace.yml +++ b/.github/workflows/fix-trailing-whitespace.yml @@ -1,9 +1,6 @@ name: Detect trailing whitespace -on: - pull_request: - branches: - - master +on: [push, pull_request] jobs: whitespace: diff --git a/.github/workflows/mingw-ci.yml b/.github/workflows/mingw-ci.yml index 44d0c05e2..009a58fd9 100644 --- a/.github/workflows/mingw-ci.yml +++ b/.github/workflows/mingw-ci.yml @@ -1,12 +1,7 @@ name: MinGW32-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] + # Important: scoop will either install 32-bit GCC or 64-bit GCC, not both. @@ -47,7 +42,7 @@ jobs: run: | Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh') scoop install sudo --global - sudo scoop install git --global + sudo scoop install git --global sudo scoop install ninja --global sudo scoop install cmake --global sudo scoop install gcc --arch 32bit --global diff --git a/.github/workflows/mingw64-ci.yml b/.github/workflows/mingw64-ci.yml index d60f50e0c..da7067098 100644 --- a/.github/workflows/mingw64-ci.yml +++ b/.github/workflows/mingw64-ci.yml @@ -1,12 +1,7 @@ name: MinGW64-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] + # Important: scoop will either install 32-bit GCC or 64-bit GCC, not both. diff --git a/.github/workflows/msys2-clang.yml b/.github/workflows/msys2-clang.yml index ea9307006..3557dd8a4 100644 --- a/.github/workflows/msys2-clang.yml +++ b/.github/workflows/msys2-clang.yml @@ -1,12 +1,7 @@ name: MSYS2-CLANG-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] + jobs: windows-mingw: diff --git a/.github/workflows/msys2.yml b/.github/workflows/msys2.yml index 49460894d..5cc483974 100644 --- a/.github/workflows/msys2.yml +++ b/.github/workflows/msys2.yml @@ -1,12 +1,6 @@ name: MSYS2-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: windows-mingw: diff --git a/.github/workflows/ubuntu18-threadsani.yml b/.github/workflows/ubuntu18-threadsani.yml index 21014aaab..ebf40f011 100644 --- a/.github/workflows/ubuntu18-threadsani.yml +++ b/.github/workflows/ubuntu18-threadsani.yml @@ -1,12 +1,7 @@ name: Ubuntu 18.04 CI (GCC 7) with Thread Sanitizer -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] + jobs: ubuntu-build: diff --git a/.github/workflows/ubuntu18.yml b/.github/workflows/ubuntu18.yml index 217a92e3f..2636bffd3 100644 --- a/.github/workflows/ubuntu18.yml +++ b/.github/workflows/ubuntu18.yml @@ -1,12 +1,6 @@ name: Ubuntu 18.04 CI (GCC 7) -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ubuntu-build: diff --git a/.github/workflows/ubuntu20-noexcept.yml b/.github/workflows/ubuntu20-noexcept.yml index e198c4b54..60c477484 100644 --- a/.github/workflows/ubuntu20-noexcept.yml +++ b/.github/workflows/ubuntu20-noexcept.yml @@ -1,12 +1,6 @@ name: Ubuntu 20.04 CI (GCC 9) without exceptions -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ubuntu-build: diff --git a/.github/workflows/ubuntu20-nothread.yml b/.github/workflows/ubuntu20-nothread.yml index d9efc1cfe..26db0f880 100644 --- a/.github/workflows/ubuntu20-nothread.yml +++ b/.github/workflows/ubuntu20-nothread.yml @@ -1,12 +1,6 @@ -name: Ubuntu 20.04 CI (GCC 9) without threads +name: Ubuntu 20.04 CI (GCC 9) Without Threads -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ubuntu-build: diff --git a/.github/workflows/ubuntu20-sani.yml b/.github/workflows/ubuntu20-sani.yml new file mode 100644 index 000000000..32687664c --- /dev/null +++ b/.github/workflows/ubuntu20-sani.yml @@ -0,0 +1,23 @@ +name: Ubuntu 20.04 CI (GCC 9) With Memory Sanitizer + +on: [push, pull_request] + +jobs: + ubuntu-build: + if: >- + ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && + ! contains(toJSON(github.event.commits.*.message), '[skip github]') + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: dependencies/.cache + key: ${{ hashFiles('dependencies/CMakeLists.txt') }} + - name: Use cmake + run: | + mkdir builddebug && + cd builddebug && + cmake -DSIMDJSON_SANITIZE=ON -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. && + cmake --build . && + ctest -j --output-on-failure -LE explicitonly \ No newline at end of file diff --git a/.github/workflows/ubuntu20-threadsani.yml b/.github/workflows/ubuntu20-threadsani.yml index 62a5bc34d..80935b814 100644 --- a/.github/workflows/ubuntu20-threadsani.yml +++ b/.github/workflows/ubuntu20-threadsani.yml @@ -1,12 +1,6 @@ name: Ubuntu 20.04 CI (GCC 9) with Thread Sanitizer -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ubuntu-build: diff --git a/.github/workflows/ubuntu20.yml b/.github/workflows/ubuntu20.yml index e79b0ff4f..1e322020c 100644 --- a/.github/workflows/ubuntu20.yml +++ b/.github/workflows/ubuntu20.yml @@ -1,12 +1,6 @@ name: Ubuntu 20.04 CI (GCC 9) -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ubuntu-build: diff --git a/.github/workflows/vs15-ci.yml b/.github/workflows/vs15-ci.yml index cc0b5e46c..8b5746b49 100644 --- a/.github/workflows/vs15-ci.yml +++ b/.github/workflows/vs15-ci.yml @@ -1,12 +1,6 @@ name: VS15-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ci: diff --git a/.github/workflows/vs16-ci.yml b/.github/workflows/vs16-ci.yml index e2c1e4566..1b6882a8b 100644 --- a/.github/workflows/vs16-ci.yml +++ b/.github/workflows/vs16-ci.yml @@ -1,12 +1,6 @@ name: VS16-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ci: diff --git a/.github/workflows/vs16-clang-ci.yml b/.github/workflows/vs16-clang-ci.yml index c2773f318..5a2ad9c30 100644 --- a/.github/workflows/vs16-clang-ci.yml +++ b/.github/workflows/vs16-clang-ci.yml @@ -1,12 +1,6 @@ name: VS16-CLANG-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ci: @@ -31,7 +25,7 @@ jobs: buildWithCMake: true cmakeAppendedArgs: -T ClangCL -DSIMDJSON_COMPETITION=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF buildWithCMakeArgs: --config Release - + - name: 'Run CTest' run: ctest -C Release -LE explicitonly --output-on-failure working-directory: "${{ github.workspace }}/../../_temp/windows" \ No newline at end of file diff --git a/.github/workflows/vs16-ninja-ci.yml b/.github/workflows/vs16-ninja-ci.yml index cf8ded038..3be83d040 100644 --- a/.github/workflows/vs16-ninja-ci.yml +++ b/.github/workflows/vs16-ninja-ci.yml @@ -1,12 +1,6 @@ name: VS16-Ninja-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ci: diff --git a/.github/workflows/vs16-noexcept-ci.yml b/.github/workflows/vs16-noexcept-ci.yml index c66f884bf..7e28608da 100644 --- a/.github/workflows/vs16-noexcept-ci.yml +++ b/.github/workflows/vs16-noexcept-ci.yml @@ -1,12 +1,6 @@ name: VS16-NoExcept-CI -on: - push: - branches: - - master - pull_request: - branches: - - master +on: [push, pull_request] jobs: ci: diff --git a/cmake/developer-options.cmake b/cmake/developer-options.cmake index 68e4431c3..c79aae3b3 100644 --- a/cmake/developer-options.cmake +++ b/cmake/developer-options.cmake @@ -3,6 +3,12 @@ # add_library(simdjson-internal-flags INTERFACE) + +option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer. The setting is unnecessary since we require padding of the inputs. You should expect tests to fail with this option turned on." OFF) +if(SIMDJSON_CHECK_EOF) + add_compile_definitions(SIMDJSON_CHECK_EOF=1) +endif() + option(SIMDJSON_SANITIZE_UNDEFINED "Sanitize undefined behavior" OFF) if(SIMDJSON_SANITIZE_UNDEFINED) add_compile_options(-fsanitize=undefined -fno-sanitize-recover=all) diff --git a/doc/basics.md b/doc/basics.md index 740a9e05a..68d12c768 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1010,7 +1010,7 @@ Unlike `parser.iterate`, `parser.iterate_many` may parse "on demand" (lazily). T As with `parser.iterate`, when calling `parser.iterate_many(string)`, no copy is made of the provided string input. The provided memory buffer may be accessed each time a JSON document is parsed. Calling `parser.iterate_many(string)` on a temporary string buffer (e.g., `docs = parser.parse_many("[1,2,3]"_padded)`) is unsafe (and will not compile) because the `document_stream` instance needs access to the buffer to return the JSON documents. -`iterate_many` can also take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents. +The `iterate_many` function can also take an optional parameter `size_t batch_size` which defines the window processing size. It is set by default to a large value (`1000000` corresponding to 1 MB). None of your JSON documents should exceed this window size, or else you will get the error `simdjson::CAPACITY`. You cannot set this window size larger than 4 GB: you will get the error `simdjson::CAPACITY`. The smaller the window size is, the less memory the function will use. Setting the window size too small (e.g., less than 100 kB) may also impact performance negatively. Leaving it to 1 MB is expected to be a good choice, unless you have some larger documents. If your documents are large (e.g., larger than a megabyte), then the `iterate_many` function is maybe ill-suited. It is really meant to support reading efficiently streams of relatively small documents (e.g., a few kilobytes each). If you have larger documents, you should use other functions like `iterate`. diff --git a/include/simdjson/arm64/numberparsing.h b/include/simdjson/arm64/numberparsing.h index eb7440d5c..b5c8f1a59 100644 --- a/include/simdjson/arm64/numberparsing.h +++ b/include/simdjson/arm64/numberparsing.h @@ -19,7 +19,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING 1 #include "simdjson/generic/numberparsing.h" diff --git a/include/simdjson/common_defs.h b/include/simdjson/common_defs.h index a694d8c83..51210e195 100644 --- a/include/simdjson/common_defs.h +++ b/include/simdjson/common_defs.h @@ -253,11 +253,8 @@ namespace std { #endif #endif -// Feature flag for partially-implemented "don't require padding" feature -// TODO remove once feature complete. -#ifndef __SIMDJSON_CHECK_EOF -# define __SIMDJSON_CHECK_EOF 1 -#endif +// The SIMDJSON_CHECK_EOF macro is a feature flag for the "don't require padding" +// feature. #if SIMDJSON_CPLUSPLUS17 // if we have C++, then fallthrough is a default attribute @@ -268,11 +265,11 @@ namespace std { #if __has_attribute(__fallthrough__) // we are good to go: # define simdjson_fallthrough __attribute__((__fallthrough__)) -#endif -#endif +#endif // __has_attribute(__fallthrough__) +#endif // SIMDJSON_CPLUSPLUS17 // on some systems, we simply do not have support for fallthrough, so use a default: #ifndef simdjson_fallthrough # define simdjson_fallthrough do {} while (0) /* fallthrough */ -#endif +#endif // simdjson_fallthrough #endif // SIMDJSON_COMMON_DEFS_H diff --git a/include/simdjson/fallback/numberparsing.h b/include/simdjson/fallback/numberparsing.h index 4513f0f4d..f33d16720 100644 --- a/include/simdjson/fallback/numberparsing.h +++ b/include/simdjson/fallback/numberparsing.h @@ -27,7 +27,8 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + #include "simdjson/generic/numberparsing.h" #endif // SIMDJSON_FALLBACK_NUMBERPARSING_H diff --git a/include/simdjson/generic/numberparsing.h b/include/simdjson/generic/numberparsing.h index 1abe524df..c58cb7818 100644 --- a/include/simdjson/generic/numberparsing.h +++ b/include/simdjson/generic/numberparsing.h @@ -367,14 +367,16 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c // the integer into a float in a lossless manner. const uint8_t *const first_after_period = p; -#ifdef SWAR_NUMBER_PARSING +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } -#endif +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } @@ -441,9 +443,7 @@ simdjson_really_inline size_t significant_digits(const uint8_t * start_digits, s // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. const uint8_t *start = start_digits; - while ((*start == '0') || (*start == '.')) { - start++; - } + while ((*start == '0') || (*start == '.')) { ++start; } // we over-decrement by one when there is a '.' return digit_count - size_t(start - start_digits); } diff --git a/include/simdjson/generic/ondemand/array-inl.h b/include/simdjson/generic/ondemand/array-inl.h index ba35724b7..1829b0ebb 100644 --- a/include/simdjson/generic/ondemand/array-inl.h +++ b/include/simdjson/generic/ondemand/array-inl.h @@ -82,7 +82,10 @@ simdjson_really_inline simdjson_result array::raw_json() noexc const uint8_t * starting_point{iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter._json_iter->peek(0)}; + // After 'consume()', we could be left pointing just beyond the document, but that + // is ok because we are not going to dereference the final pointer position, we just + // use it to compute the length in bytes. + const uint8_t * final_point{iter._json_iter->unsafe_pointer()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } diff --git a/include/simdjson/generic/ondemand/document-inl.h b/include/simdjson/generic/ondemand/document-inl.h index 045411d50..1f9259dc2 100644 --- a/include/simdjson/generic/ondemand/document-inl.h +++ b/include/simdjson/generic/ondemand/document-inl.h @@ -156,12 +156,14 @@ simdjson_really_inline error_code document::consume() noexcept { } simdjson_really_inline simdjson_result document::raw_json() noexcept { - printf("document::raw_json()\n"); auto _iter = get_root_value_iterator(); const uint8_t * starting_point{_iter.peek_start()}; auto error = consume(); if(error) { return error; } - const uint8_t * final_point{iter.peek(0)}; + // After 'consume()', we could be left pointing just beyond the document, but that + // is ok because we are not going to dereference the final pointer position, we just + // use it to compute the length in bytes. + const uint8_t * final_point{iter.unsafe_pointer()}; return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); } diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h index 209cd9985..d7e88ddb9 100644 --- a/include/simdjson/generic/ondemand/json_iterator-inl.h +++ b/include/simdjson/generic/ondemand/json_iterator-inl.h @@ -35,7 +35,9 @@ simdjson_really_inline json_iterator::json_iterator(const uint8_t *buf, ondemand { logger::log_headers(); +#if SIMDJSON_CHECK_EOF assert_more_tokens(); +#endif } inline void json_iterator::rewind() noexcept { @@ -84,10 +86,10 @@ simdjson_warn_unused simdjson_really_inline error_code json_iterator::skip_child logger::log_end_value(*this, "skip"); _depth--; if (depth() <= parent_depth) { return SUCCESS; } -#if __SIMDJSON_CHECK_EOF +#if SIMDJSON_CHECK_EOF // If there are no more tokens, the parent is incomplete. if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); } -#endif // __SIMDJSON_CHECK_EOF +#endif // SIMDJSON_CHECK_EOF break; /*case '"': if(*peek() == ':') { @@ -199,23 +201,28 @@ simdjson_really_inline void json_iterator::abandon() noexcept { } simdjson_really_inline const uint8_t *json_iterator::return_current_and_advance() noexcept { - // The following assert_more_tokens is currently disabled because rely on end-of-file buffering. - // assert_more_tokens(); - // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF - // is ON by default, we have no choice but to disable it for real with a comment. +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif // SIMDJSON_CHECK_EOF return token.return_current_and_advance(); } +simdjson_really_inline const uint8_t *json_iterator::unsafe_pointer() const noexcept { + // deliberately done without safety guard: + return token.peek(0); +} + simdjson_really_inline const uint8_t *json_iterator::peek(int32_t delta) const noexcept { - // The following assert_more_tokens is currently disabled because rely on end-of-file buffering. - // assert_more_tokens(delta+1); - // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF - // is ON by default, we have no choice but to disable it for real with a comment. +#if SIMDJSON_CHECK_EOF + assert_more_tokens(delta+1); +#endif // SIMDJSON_CHECK_EOF return token.peek(delta); } simdjson_really_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept { +#if SIMDJSON_CHECK_EOF assert_more_tokens(delta+1); +#endif // #if SIMDJSON_CHECK_EOF return token.peek_length(delta); } @@ -223,17 +230,15 @@ simdjson_really_inline const uint8_t *json_iterator::peek(token_position positio // todo: currently we require end-of-string buffering, but the following // assert_valid_position should be turned on if/when we lift that condition. // assert_valid_position(position); - // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF + // This is almost surely related to SIMDJSON_CHECK_EOF but given that SIMDJSON_CHECK_EOF // is ON by default, we have no choice but to disable it for real with a comment. return token.peek(position); } simdjson_really_inline uint32_t json_iterator::peek_length(token_position position) const noexcept { - // todo: currently we require end-of-string buffering, but the following - // assert_valid_position should be turned on if/when we lift that condition. - // assert_valid_position(position); - // This is almost surely related to __SIMDJSON_CHECK_EOF but given that __SIMDJSON_CHECK_EOF - // is ON by default, we have no choice but to disable it for real with a comment. +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } diff --git a/include/simdjson/generic/ondemand/json_iterator.h b/include/simdjson/generic/ondemand/json_iterator.h index 9165c2d53..6f2876c7e 100644 --- a/include/simdjson/generic/ondemand/json_iterator.h +++ b/include/simdjson/generic/ondemand/json_iterator.h @@ -145,12 +145,21 @@ public: * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. */ simdjson_really_inline uint32_t peek_length(int32_t delta=0) const noexcept; + /** + * Get a pointer to the current location in the input buffer. + * + * This is not null-terminated; it is a view into the JSON. + * + * You may be pointing outside of the input buffer: it is not generally + * safe to derefence this pointer. + */ + simdjson_really_inline const uint8_t *unsafe_pointer() const noexcept; /** * Get the JSON text for a given token. * * This is not null-terminated; it is a view into the JSON. * - * @param index The position of the token to retrieve. + * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when * it isn't used ... @@ -161,7 +170,7 @@ public: * * The length will include any whitespace at the end of the token. * - * @param index The position of the token to retrieve. + * @param position The position of the token to retrieve. */ simdjson_really_inline uint32_t peek_length(token_position position) const noexcept; /** diff --git a/include/simdjson/generic/ondemand/parser.h b/include/simdjson/generic/ondemand/parser.h index fa54c7f4e..38a74c99f 100644 --- a/include/simdjson/generic/ondemand/parser.h +++ b/include/simdjson/generic/ondemand/parser.h @@ -129,8 +129,6 @@ public: * those bytes are initialized to, as long as they are allocated. * * @param json The JSON to parse. - * @param len The length of the JSON. - * @param allocated The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING). * * @return The iterator, or an error: * - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes. @@ -143,6 +141,67 @@ public: */ simdjson_warn_unused simdjson_result iterate_raw(padded_string_view json) & noexcept; + + /** + * Parse a buffer containing many JSON documents. + * + * auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded; + * ondemand::parser parser; + * ondemand::document_stream docs = parser.iterate_many(json); + * for (auto & doc : docs) { + * std::cout << doc["foo"] << std::endl; + * } + * // Prints 1 2 3 + * + * No copy of the input buffer is made. + * + * The function is lazy: it may be that no more than one JSON document at a time is parsed. + * + * The caller is responsabile to ensure that the input string data remains unchanged and is + * not deleted during the loop. + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with whitespace. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excesively small values may impact negatively the + * performance. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The concatenated JSON to parse. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept; diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 258e75cf4..a18296b73 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -46,10 +46,10 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator } simdjson_warn_unused simdjson_really_inline error_code value_iterator::end_container() noexcept { -#if __SIMDJSON_CHECK_EOF +#if SIMDJSON_CHECK_EOF if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); } // if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); } -#endif // __SIMDJSON_CHECK_EOF +#endif // SIMDJSON_CHECK_EOF _json_iter->ascend_to(depth()-1); return SUCCESS; } @@ -130,6 +130,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator // Get the key and colon, stopping at the value. raw_json_string actual_key; // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. // field_key() advances the pointer and checks that '"' is found (corresponding to a key). // The depth is left unchanged by field_key(). if ((error = field_key().get(actual_key) )) { abandon(); return error; }; @@ -260,6 +261,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator // Get the key and colon, stopping at the value. raw_json_string actual_key; // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. // field_key() advances the pointer and checks that '"' is found (corresponding to a key). // The depth is left unchanged by field_key(). if ((error = field_key().get(actual_key) )) { abandon(); return error; }; @@ -309,6 +311,7 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator // Get the key and colon, stopping at the value. raw_json_string actual_key; // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. // field_key() advances the pointer and checks that '"' is found (corresponding to a key). // The depth is left unchanged by field_key(). error = field_key().get(actual_key); SIMDJSON_ASSUME(!error); diff --git a/include/simdjson/haswell/numberparsing.h b/include/simdjson/haswell/numberparsing.h index ee1ce1cad..71b5b3ad5 100644 --- a/include/simdjson/haswell/numberparsing.h +++ b/include/simdjson/haswell/numberparsing.h @@ -27,7 +27,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING 1 #include "simdjson/generic/numberparsing.h" diff --git a/include/simdjson/ppc64/numberparsing.h b/include/simdjson/ppc64/numberparsing.h index 783dd99c0..184e7a70d 100644 --- a/include/simdjson/ppc64/numberparsing.h +++ b/include/simdjson/ppc64/numberparsing.h @@ -25,7 +25,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) { } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING 1 #include "simdjson/generic/numberparsing.h" diff --git a/include/simdjson/westmere/numberparsing.h b/include/simdjson/westmere/numberparsing.h index 6556b1577..35f5f980b 100644 --- a/include/simdjson/westmere/numberparsing.h +++ b/include/simdjson/westmere/numberparsing.h @@ -27,7 +27,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING 1 #include "simdjson/generic/numberparsing.h" diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 8ec76c9bc..b54de37d2 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -10386,7 +10386,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace arm64 } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING /* begin file include/simdjson/generic/numberparsing.h */ #include @@ -10743,7 +10743,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c // the integer into a float in a lossless manner. const uint8_t *const first_after_period = p; -#ifdef SWAR_NUMBER_PARSING +#ifdef SIMDJSON_SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. if (is_made_of_eight_digits_fast(p)) { @@ -11826,7 +11826,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace fallback } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING /* begin file include/simdjson/generic/numberparsing.h */ #include @@ -12182,7 +12182,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c // the integer into a float in a lossless manner. const uint8_t *const first_after_period = p; -#ifdef SWAR_NUMBER_PARSING +#ifdef SIMDJSON_SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. if (is_made_of_eight_digits_fast(p)) { @@ -13749,7 +13749,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace haswell } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING /* begin file include/simdjson/generic/numberparsing.h */ #include @@ -14106,7 +14106,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c // the integer into a float in a lossless manner. const uint8_t *const first_after_period = p; -#ifdef SWAR_NUMBER_PARSING +#ifdef SIMDJSON_SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. if (is_made_of_eight_digits_fast(p)) { @@ -15772,7 +15772,7 @@ parse_eight_digits_unrolled(const uint8_t *chars) { } // namespace ppc64 } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING /* begin file include/simdjson/generic/numberparsing.h */ #include @@ -16129,7 +16129,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c // the integer into a float in a lossless manner. const uint8_t *const first_after_period = p; -#ifdef SWAR_NUMBER_PARSING +#ifdef SIMDJSON_SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. if (is_made_of_eight_digits_fast(p)) { @@ -17653,7 +17653,7 @@ static simdjson_really_inline uint32_t parse_eight_digits_unrolled(const uint8_t } // namespace westmere } // namespace simdjson -#define SWAR_NUMBER_PARSING +#define SIMDJSON_SWAR_NUMBER_PARSING /* begin file include/simdjson/generic/numberparsing.h */ #include @@ -18010,7 +18010,7 @@ simdjson_really_inline error_code parse_decimal(simdjson_unused const uint8_t *c // the integer into a float in a lossless manner. const uint8_t *const first_after_period = p; -#ifdef SWAR_NUMBER_PARSING +#ifdef SIMDJSON_SWAR_NUMBER_PARSING // this helps if we have lots of decimals! // this turns out to be frequent enough. if (is_made_of_eight_digits_fast(p)) { diff --git a/tests/dom/basictests.cpp b/tests/dom/basictests.cpp index 6e4228e18..29db125e4 100644 --- a/tests/dom/basictests.cpp +++ b/tests/dom/basictests.cpp @@ -2116,7 +2116,20 @@ namespace to_string_tests { } } - +bool simple_overflows() { + std::cout << "Running " << __func__ << std::endl; + simdjson::dom::parser parser; + simdjson::dom::element doc; + ASSERT_ERROR( parser.parse(std::string("[f]")).get(doc), simdjson::F_ATOM_ERROR); + ASSERT_ERROR( parser.parse(std::string("[t]")).get(doc), simdjson::T_ATOM_ERROR); + ASSERT_ERROR( parser.parse(std::string("[n]")).get(doc), simdjson::N_ATOM_ERROR); + ASSERT_ERROR( parser.parse(std::string("[-]")).get(doc), simdjson::NUMBER_ERROR); + ASSERT_ERROR( parser.parse(std::string("{\"a\":f}")).get(doc), simdjson::F_ATOM_ERROR); + ASSERT_ERROR( parser.parse(std::string("{\"a\":t}")).get(doc), simdjson::T_ATOM_ERROR); + ASSERT_ERROR( parser.parse(std::string("{\"a\":n}")).get(doc), simdjson::N_ATOM_ERROR); + ASSERT_ERROR( parser.parse(std::string("{\"a\":-}")).get(doc), simdjson::NUMBER_ERROR); + return true; +} int main(int argc, char *argv[]) { std::cout << std::unitbuf; @@ -2141,7 +2154,6 @@ int main(int argc, char *argv[]) { return EXIT_FAILURE; } } - // this is put here deliberately to check that the documentation is correct (README), // should this fail to compile, you should update the documentation: if (simdjson::active_implementation->name() == "unsupported") { @@ -2153,7 +2165,8 @@ int main(int argc, char *argv[]) { std::cout << "------------------------------------------------------------" << std::endl; std::cout << "Running basic tests." << std::endl; - if (to_string_tests::run() && + if (simple_overflows() && + to_string_tests::run() && validate_tests::run() && minify_tests::run() && parse_api_tests::run() && diff --git a/tests/dom/document_stream_tests.cpp b/tests/dom/document_stream_tests.cpp index a1639f04d..c9ab51827 100644 --- a/tests/dom/document_stream_tests.cpp +++ b/tests/dom/document_stream_tests.cpp @@ -735,7 +735,7 @@ namespace document_stream_tests { bool document_stream_test() { std::cout << "Running " << __func__ << std::endl; fflush(NULL); - const size_t n_records = 10000; + const size_t n_records = 100; std::string data; std::vector buf(1024); for (size_t i = 0; i < n_records; ++i) { @@ -775,7 +775,7 @@ namespace document_stream_tests { bool document_stream_utf8_test() { std::cout << "Running " << __func__ << std::endl; fflush(NULL); - const size_t n_records = 10000; + const size_t n_records = 100; std::string data; std::vector buf(1024); for (size_t i = 0; i < n_records; ++i) { diff --git a/tests/dom/document_tests.cpp b/tests/dom/document_tests.cpp index 8ee4122b3..05f3bd4ba 100644 --- a/tests/dom/document_tests.cpp +++ b/tests/dom/document_tests.cpp @@ -108,7 +108,7 @@ namespace document_tests { // returns true if successful bool skyprophet_test() { std::cout << "Running " << __func__ << std::endl; - const size_t n_records = 100000; + const size_t n_records = 100; std::vector data; std::vector buf(1024); for (size_t i = 0; i < n_records; ++i) { diff --git a/tests/ondemand/ondemand_array_error_tests.cpp b/tests/ondemand/ondemand_array_error_tests.cpp index 0822f7de2..2625d30d9 100644 --- a/tests/ondemand/ondemand_array_error_tests.cpp +++ b/tests/ondemand/ondemand_array_error_tests.cpp @@ -100,10 +100,16 @@ namespace array_error_tests { } bool array_iterate_incomplete_error() { TEST_START(); +#if SIMDJSON_CHECK_EOF ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT })); ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); ONDEMAND_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCOMPLETE_ARRAY_OR_OBJECT })); ONDEMAND_SUBTEST("unclosed after array", R"([ [])", assert_iterate(doc.get_array().at(0), { INCOMPLETE_ARRAY_OR_OBJECT })); +#else + ONDEMAND_SUBTEST("unclosed after array", R"([ [1] )", assert_iterate(doc.get_array().at(0), { int64_t(1) })); + ONDEMAND_SUBTEST("unclosed after array", R"([ [1,])", assert_iterate(doc.get_array().at(0), { int64_t(1) }, { INCORRECT_TYPE, TAPE_ERROR })); + ONDEMAND_SUBTEST("unclosed after array", R"([ [1])", assert_iterate(doc.get_array().at(0), { int64_t(1) })); +#endif TEST_SUCCEED(); } diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp index 8bfa76691..48f6d4176 100644 --- a/tests/ondemand/ondemand_document_stream_tests.cpp +++ b/tests/ondemand/ondemand_document_stream_tests.cpp @@ -315,7 +315,7 @@ namespace document_stream_tests { bool document_stream_test() { TEST_START(); fflush(NULL); - const size_t n_records = 10000; + const size_t n_records = 100; std::string data; std::vector buf(1024); // Generating data @@ -352,7 +352,7 @@ namespace document_stream_tests { bool document_stream_utf8_test() { TEST_START(); fflush(NULL); - const size_t n_records = 10000; + const size_t n_records = 100; std::string data; std::vector buf(1024); // Generating data diff --git a/tests/ondemand/ondemand_object_error_tests.cpp b/tests/ondemand/ondemand_object_error_tests.cpp index 18de0d719..1059e0354 100644 --- a/tests/ondemand/ondemand_object_error_tests.cpp +++ b/tests/ondemand/ondemand_object_error_tests.cpp @@ -88,10 +88,10 @@ namespace object_error_tests { } bool object_lookup_unclosed_error() { TEST_START(); - #if __SIMDJSON_CHECK_EOF + #if SIMDJSON_CHECK_EOF ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); #else - ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_success(doc["a"])); + ONDEMAND_SUBTEST("unclosed", R"({ "a": )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); #endif ONDEMAND_SUBTEST("unclosed", R"({ "a" )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT)); ONDEMAND_SUBTEST("unclosed", R"({ )", assert_error(doc["a"], INCOMPLETE_ARRAY_OR_OBJECT));