Compare commits

...

25 Commits

Author SHA1 Message Date
Daniel Lemire 8ad452459c moving the files back to ondemand for now. 2025-07-14 11:46:06 -04:00
Daniel Lemire d987f1871c Initial work on JSON builder 2025-07-14 11:45:25 -04:00
Daniel Lemire 4acaf2ea1c improving DOM ranges test and silencing a warning (#2385)
* improving DOM ranges test and silencing a warning

* moving test_main to macros (even though it is not a macro)

* moving test_main
2025-07-04 15:46:50 -04:00
Daniel Lemire 86adda06d0 clarifying NDEBUG usage (#2388)
* clarifying NDEBUG usage

* init
2025-07-04 15:46:37 -04:00
Daniel Lemire 8df5f96eba guarding the POSITION_INDEPENDENT_CODE (#2386) 2025-06-30 16:41:49 -04:00
xkszltl 937667796c Replace -fPIC with POSITION_INDEPENDENT_CODE. (#2383) 2025-06-30 11:26:27 -04:00
Cuda Chen 7d86b04247 Add document and tests of std::ranges support of DOM API (#2381)
* Add document and tests of std::ranges support of DOM API

* Alter lambda

* Fix require of range
2025-06-24 15:13:44 -04:00
Daniel Lemire 265022ad8f Fixing MSYS2 issue (#2380)
* Google Benchmarks does not support MSYS2.

* narrowing it down
2025-06-21 21:26:52 -04:00
Daniel Lemire 0c0ce1bd48 3.13.0 2025-06-04 00:46:03 -04:00
Daniel Lemire f64c004cb7 making it easier to convert to std::string (#2378)
* making it easier to convert to std::string

* typo

* guarding C++20

* bad semi-colon
2025-06-04 00:22:10 -04:00
Daniel Lemire d7d19db6ae Emscripten (#2377)
* emscripten

* cmake_policy

* space
2025-06-03 09:10:59 -04:00
Daniel Lemire a260c967ed adding test for issue 2375 (#2376)
* adding test for issue 2375

* update

* removing space
2025-06-02 09:24:40 -04:00
Daniel Lemire 16e390d81a fix for issue 2373 (#2374)
* fix for issue 2373

* bumping google bench

* removing json11 since dropbox retired it

* removing json11 as a dependency
2025-05-31 00:56:19 -04:00
Daniel Lemire 8d34e14000 Update dom.md 2025-05-31 00:27:28 -04:00
shuiyisong b717136fd9 chore: add greptimedb in real world usage (#2372) 2025-05-07 16:33:24 -04:00
Daniel Lemire f3ac74caf8 slightly better documemntation of the C++20 features (#2371)
* slightly better documemntation of the C++20 features

* adding another remark
2025-04-23 18:32:51 -04:00
Daniel Lemire 2369927661 Moving from Ubuntu 20 to 24 (#2366) 2025-04-18 17:56:46 -04:00
Daniel Lemire 818c0491a1 Treat -0 as -0.0 when SIMDJSON_MINUS_ZERO_AS_FLOAT is set (#2364)
* In the DOM API, treat -0 as -0.0

* documenting...

* adding it to On-Demand
2025-04-18 13:56:02 -04:00
Daniel Lemire 9f14f90a64 [documentation] We run on practically all systems. (#2365)
* We run on practically all systems.

* Update doc/basics.md

Co-authored-by: Antoine Pitrou <pitrou@free.fr>

---------

Co-authored-by: Antoine Pitrou <pitrou@free.fr>
2025-04-18 13:48:13 -04:00
Daniel Lemire 00843d2711 better documentation 2025-04-16 11:45:52 -04:00
Daniel Lemire 2887a17bab bumping up nlohmann/json 2025-04-13 14:43:30 -04:00
huangqinjin d84c934768 do not try to include <string_view> without C++17 for MSVC (#2360)
Doing that will emit warning at https://github.com/microsoft/STL/blob/vs-2022-17.13/stl/inc/string_view#L12.
2025-03-28 19:23:15 -04:00
huangqinjin 7cec7c7ae4 fix VS2017 exception detection (#2359)
MSVC option /EHsc defines __cpp_exceptions only in VS2019 and above,
_CPPUNWIND should be used before VS2019.

https://learn.microsoft.com/en-us/cpp/preprocessor/predefined-macros#microsoft-specific-predefined-macros
2025-03-28 19:22:56 -04:00
Daniel Lemire c52b010a57 missing string_view include (#2361) 2025-03-28 19:22:38 -04:00
Daniel Lemire ee301599b1 Update basics.md 2025-03-28 11:53:07 -04:00
46 changed files with 2795 additions and 260 deletions
+17
View File
@@ -0,0 +1,17 @@
on: [push, pull_request]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
- uses: mymindstorm/setup-emsdk@6ab9eb1bda2574c4ddb79809fc9247783eaf9021 # v14
- name: Verify
run: emcc -v
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v3.6.0
- name: Configure
run: emcmake cmake -B build
- name: Build # We build but do not test
run: cmake --build build
@@ -4,7 +4,7 @@ on: [push, pull_request]
jobs:
whitespace:
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- name: Remove whitespace and check the diff
-38
View File
@@ -1,38 +0,0 @@
name: Ubuntu 20.04 CI (GCC 8)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
env:
CXX: g++-8
CC: gcc-8
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Install GCC 8
run: sudo apt-get install -y g++-8
- name: Use cmake
run: |
mkdir builddebug &&
cd builddebug &&
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
cmake --build . &&
ctest --output-on-failure -LE explicitonly -j &&
cd .. &&
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest --output-on-failure -LE explicitonly -j &&
cmake --install . &&
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
cd ../tests/installation_tests/find &&
mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build .
-47
View File
@@ -1,47 +0,0 @@
name: Ubuntu 20.04 CI (GCC 9)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with:
path: dependencies/.cache
key: ${{ hashFiles('dependencies/CMakeLists.txt') }}
- name: Use cmake to build just the library
run: |
mkdir buildjustlib &&
cd buildjustlib &&
cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DSIMDJSON_DEVELOPER_MODE=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
cmake --install . &&
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp &&
c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson &&
cd ../tests/installation_tests/find &&
mkdir buildjustlib &&
cd buildjustlib &&
cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../buildjustlib/destination .. &&
cmake --build .
- name: Use cmake
run: |
mkdir builddebug &&
cd builddebug &&
cmake -DCMAKE_BUILD_TYPE=Debug -DSIMDJSON_GOOGLE_BENCHMARKS=OFF -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF .. &&
cmake --build . &&
ctest --output-on-failure -LE explicitonly -j &&
cd .. &&
mkdir build &&
cd build &&
cmake -DSIMDJSON_GOOGLE_BENCHMARKS=ON -DSIMDJSON_DEVELOPER_MODE=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX:PATH=destination .. &&
cmake --build . &&
ctest --output-on-failure -LE explicitonly -j &&
cmake --install . &&
echo -e '#include <simdjson.h>\nint main(int argc,char**argv) {simdjson::dom::parser parser;simdjson::dom::element tweets = parser.load(argv[1]); }' > tmp.cpp && c++ -Idestination/include -Ldestination/lib -std=c++17 -Wl,-rpath,destination/lib -o linkandrun tmp.cpp -lsimdjson && ./linkandrun jsonexamples/twitter.json &&
cd ../tests/installation_tests/find &&
mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build .
@@ -13,7 +13,7 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
@@ -7,7 +7,7 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
@@ -7,7 +7,7 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
@@ -7,7 +7,7 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
@@ -25,7 +25,7 @@ jobs:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-20.04
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
+12 -4
View File
@@ -1,9 +1,11 @@
cmake_minimum_required(VERSION 3.14)
cmake_policy(VERSION 3.5) # For doctest
project(
simdjson
# The version number is modified by tools/release.py
VERSION 3.12.3
VERSION 3.13.0
DESCRIPTION "Parsing gigabytes of JSON per second"
HOMEPAGE_URL "https://simdjson.org/"
LANGUAGES CXX C
@@ -20,8 +22,8 @@ string(
# ---- Options, variables ----
# These version numbers are modified by tools/release.py
set(SIMDJSON_LIB_VERSION "25.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "25" CACHE STRING "simdjson library soversion")
set(SIMDJSON_LIB_VERSION "26.0.0" CACHE STRING "simdjson library version")
set(SIMDJSON_LIB_SOVERSION "26" CACHE STRING "simdjson library soversion")
option(SIMDJSON_BUILD_STATIC_LIB "Build simdjson_static library along with simdjson (only makes sense if BUILD_SHARED_LIBS=ON)" OFF)
if(SIMDJSON_BUILD_STATIC_LIB AND NOT BUILD_SHARED_LIBS)
@@ -84,7 +86,7 @@ set_target_properties(
)
# FIXME: Use proper CMake integration for exports
if(MSVC AND BUILD_SHARED_LIBS)
if(WIN32 AND BUILD_SHARED_LIBS)
target_compile_definitions(
simdjson
PRIVATE SIMDJSON_BUILDING_WINDOWS_DYNAMIC_LIBRARY=1
@@ -111,6 +113,12 @@ if(
)
endif()
option(SIMDJSON_MINUS_ZERO_AS_FLOAT "Treat -0 as a floating-point value" OFF)
if(SIMDJSON_MINUS_ZERO_AS_FLOAT)
simdjson_add_props(target_compile_definitions PRIVATE SIMDJSON_MINUS_ZERO_AS_FLOAT=1)
endif(SIMDJSON_MINUS_ZERO_AS_FLOAT)
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(loongarch64)$")
option(SIMDJSON_PREFER_LSX "Prefer LoongArch SX" ON)
include(CheckCXXCompilerFlag)
+1 -1
View File
@@ -38,7 +38,7 @@ PROJECT_NAME = simdjson
# could be handy for archiving the generated documentation or if some version
# control system is used.
PROJECT_NUMBER = "3.12.3"
PROJECT_NUMBER = "3.13.0"
# Using the PROJECT_BRIEF tag one can provide an optional one line description
# for a project that appears at the top of each page and should give viewer a
+1
View File
@@ -63,6 +63,7 @@ Real-world usage
- [fastgron](https://github.com/adamritter/fastgron)
- [WasmEdge](https://wasmedge.org)
- [RonDB](https://github.com/logicalclocks/rondb)
- [GreptimeDB](https://github.com/GreptimeTeam/greptimedb)
If you are planning to use simdjson in a product, please work from one of our releases.
+4 -5
View File
@@ -2,7 +2,10 @@
# Flags used by exes and by the simdjson library (project-wide flags)
#
add_library(simdjson-internal-flags INTERFACE)
if(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE)
# We default to ON for all targets, so that we can use the library in shared libraries.
set_target_properties(simdjson-internal-flags PROPERTIES POSITION_INDEPENDENT_CODE ON)
endif(NOT DEFINED CMAKE_POSITION_INDEPENDENT_CODE)
option(SIMDJSON_CHECK_EOF "Check for the end of the input buffer. The setting is unnecessary since we require padding of the inputs. You should expect tests to fail with this option turned on." OFF)
if(SIMDJSON_CHECK_EOF)
@@ -52,7 +55,6 @@ endif()
option(SIMDJSON_SANITIZE_MEMORY "Sanitize memory" OFF)
if(SIMDJSON_SANITIZE_MEMORY)
message(STATUS "Setting the memory sanitizer.")
add_compile_options(
@@ -158,9 +160,6 @@ We recommend Visual Studio 2019 or better on a 64-bit system.")
add_compile_options(/Zi)
endif()
else()
if(NOT WIN32)
target_compile_options(simdjson-internal-flags INTERFACE -fPIC)
endif()
target_compile_options(
simdjson-internal-flags INTERFACE
-Werror -Wall -Wextra -Weffc++ -Wsign-compare -Wshadow -Wwrite-strings
+5 -13
View File
@@ -7,12 +7,12 @@ option(SIMDJSON_ALLOW_DOWNLOADS
cmake_dependent_option(SIMDJSON_COMPETITION "Compile competitive benchmarks" ON
SIMDJSON_ALLOW_DOWNLOADS OFF)
cmake_dependent_option(SIMDJSON_GOOGLE_BENCHMARKS "compile the Google Benchmark benchmarks" ON
SIMDJSON_ALLOW_DOWNLOADS OFF)
"SIMDJSON_ALLOW_DOWNLOADS OR MINGW" OFF)
if(SIMDJSON_GOOGLE_BENCHMARKS)
CPMAddPackage(
NAME google_benchmarks
URL https://github.com/google/benchmark/archive/refs/tags/v1.7.1.zip
URL https://github.com/google/benchmark/archive/refs/tags/v1.9.4.zip
OPTIONS
"BENCHMARK_ENABLE_TESTING OFF"
"BENCHMARK_ENABLE_INSTALL OFF"
@@ -22,7 +22,7 @@ endif()
CPMAddPackage(
NAME simdjson-data
URL https://github.com/simdjson/simdjson-data/archive/a5b13babe65c1bba7186b41b43d4cbdc20a5c470.zip
URL https://github.com/simdjson/simdjson-data/archive/351949906abde446f0314bf79606fb5d884f5be7.zip
)
option(SIMDJSON_USE_BOOST_JSON "Try to include BOOST_JSON, this may break your binaries under some systems." OFF)
@@ -93,19 +93,11 @@ int main() {}
CPMAddPackage(
NAME nlohmann_json
URL https://github.com/nlohmann/json/archive/refs/tags/v3.10.5.zip
URL https://github.com/nlohmann/json/archive/refs/tags/v3.12.0.zip
)
set_property(TARGET nlohmann_json APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS SIMDJSON_COMPETITION_NLOHMANN_JSON)
CPMAddPackage(
NAME json11
URL https://github.com/dropbox/json11/archive/ec4e45219af1d7cde3d58b49ed762376fccf1ace.zip
DOWNLOAD_ONLY YES
)
add_library(json11 STATIC "${json11_SOURCE_DIR}/json11.cpp")
target_include_directories(json11 SYSTEM PUBLIC "${json11_SOURCE_DIR}")
target_compile_definitions(json11 INTERFACE SIMDJSON_COMPETITION_JSON11)
set(jsoncpp_SOURCE_DIR "${simdjson_SOURCE_DIR}/dependencies/jsoncppdist")
add_library(jsoncpp STATIC "${jsoncpp_SOURCE_DIR}/jsoncpp.cpp")
@@ -177,7 +169,7 @@ int main() {}
endif()
add_library(competition-all INTERFACE)
target_link_libraries(competition-all INTERFACE competition-core jsoncpp json11 fastjson gason ujson4c)
target_link_libraries(competition-all INTERFACE competition-core jsoncpp fastjson gason ujson4c)
endfunction()
if(SIMDJSON_COMPETITION)
+63 -11
View File
@@ -47,13 +47,15 @@ An overview of what you need to know to use simdjson, with examples.
Requirements
------------------
- A recent compiler (LLVM clang 6 or better, GNU GCC 7.4 or better, Xcode 11 or better) on a 64-bit (PPC, ARM or x64 Intel/AMD) POSIX systems such as macOS, freeBSD or Linux. We require that the compiler supports the C++11 standard or better.
- Visual Studio 2017 or better under 64-bit Windows. Users should target a 64-bit build (x64 or ARM64) instead of a 32-bit build (x86). We support the LLVM clang compiler under Visual Studio (clang-cl) as well as as the regular Visual Studio compiler. For better release performance (both compile time and execution time), we recommend Visual Studio users adopt LLVM (clang-cl). We also support MinGW 64-bit under Windows.
The simdjson library is widely deployed in popular systems such as the Node.js runtime
environment.
- A recent compiler (LLVM clang 6 or better, GNU GCC 7.4 or better, Xcode 11 or better) on POSIX systems such as macOS, FreeBSD or Linux. We require that the compiler supports the C++11 standard or better. We test the library on a big-endian system (IBM s390x with Linux).
- Visual Studio 2017 or better. We support the LLVM clang compiler under Visual Studio (clang-cl) as well as as the regular Visual Studio compiler. For better release performance (both compile time and execution time), we recommend Visual Studio users adopt LLVM (clang-cl). We also support MinGW 64-bit under Windows.
Support for AVX-512 require a processor with AVX512-VBMI2 support (Ice Lake or better, AMD Zen 4 or better) under a 64-bit system and a recent compiler (LLVM clang 6 or better, GCC 8 or better, Visual Studio 2019 or better). You need a correspondingly recent assembler such as gas (2.30+) or nasm (2.14+): recent compilers usually come with recent assemblers. If you mix a recent compiler with an incompatible/old assembler (e.g., when using a recent compiler with an old Linux distribution), you may get errors at build time because the compiler produces instructions that the assembler does not recognize: you should update your assembler to match your compiler (e.g., upgrade binutils to version 2.30 or better under Linux) or use an older compiler matching the capabilities of your assembler.
We test the library on a big-endian system (IBM s390x with Linux).
Including simdjson
------------------
@@ -320,13 +322,15 @@ transcode the UTF-8 strings produced by the simdjson library to other formats. S
Avoiding pitfalls: enable development checks
--------------------
We recommend that you first compile and run your code in Debug mode:
We recommend that you first compile and run your code in debug mode:
- under Visual Studio, it means having the `_DEBUG` macro defined,
- for other compilers, it means leaving the `__OPTIMIZE__` macro undefined.
- for many other compilers, it means leaving the `__OPTIMIZE__` macro undefined.
The simdjson code will set `SIMDJSON_DEVELOPMENT_CHECKS=1` in debug mode.
Alternatively, you can set the macro `SIMDJSON_DEVELOPMENT_CHECKS` to 1 prior to including
The simdjson code will set `SIMDJSON_DEVELOPMENT_CHECKS=1` in debug mode. Because
the C++ standard does not provide a direct way of checking for a debug build, and
because you may want the checks while building with otimizations, you can set
the macro `SIMDJSON_DEVELOPMENT_CHECKS` to 1 prior to including
the `simdjson.h` header to enable these additional checks: just make sure you remove the
definition once your code has been tested. When `SIMDJSON_DEVELOPMENT_CHECKS` is set to 1, the
simdjson library runs additional (expensive) tests on your code to help ensure that you are
@@ -335,9 +339,12 @@ using the library in a safe manner.
Once your code has been tested, you can then run it in
Release mode: under Visual Studio, it means having the `_DEBUG` macro undefined, and, for other
compilers, it means setting `__OPTIMIZE__` to a positive integer. You can also forcefully
disable these checks by setting `SIMDJSON_DEVELOPMENT_CHECKS` to 0. Once your code is tested, we
further encourage you to define `NDEBUG` in your Release builds to disable additional runtime
testing and get the best performance.
disable these checks by setting `SIMDJSON_DEVELOPMENT_CHECKS` to 0.
Once your code is tested, we further encourage you to define `NDEBUG` in your release
builds to disable additional runtime testing and get the best performance. We
disable these checks on a best-effort basis but the C++ standard does not provide
a direct way to check for a release build.
Using the parsed JSON
---------------------
@@ -1081,6 +1088,32 @@ int main(void) {
### 2. Use `tag_invoke` for custom types (C++20)
The simdjson library takes advantage of C++20. An immediate benefit
is that you can deserialize JSON data directly in standard containers
and other standard value types:
```C++
simdjson::padded_string json = R"({"data" : [1,2,3,4]})"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document d = parser.iterate(json);
std::vector<uint8_t> array = d["data"].get<std::vector<uint8_t>>();
```
Appending to an existing container is just as easy:
```C++
std::vector<uint32_t> array = {0, 0};
simdjson::padded_string json = R"({"data" : [1,2,3,4]})"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document d = parser.iterate(json);
d["data"].get<std::vector<uint32_t>>(array);
// array is now {0,0,1,2,3,4}
```
In C++20, the standard introduced the notion of *customization point*.
A customization point is a function or function object that can be customized for different types. It allows library authors to provide default behavior while giving users the ability to override this behavior for specific types.
@@ -1257,7 +1290,7 @@ You may also conditionally fill in `std::optional` values.
// error is simdjson::SUCCESS
```
You can also deserialized to map-like types with keys that can be constructed
You can also deserialize to map-like types with keys that can be constructed
from `std::string_view` instances:
@@ -2246,6 +2279,11 @@ Thus it is a dynamically typed number. Before accessing the value, you must dete
such a number of `get_number()`, you get the error `BIGINT_ERROR`. You can access the underlying string of digits with the function `raw_json_token()` which returns a `std::string_view` instance starting at the beginning of the digit. You can also call `get_double()` to get a floating-point approximation.
By default, the string `-0` is parsed as the integer 0 as in Python or C++. If you set the macro
`SIMDJSON_MINUS_ZERO_AS_FLOAT` to `1` when building simdjson, you can get that `-0` is mapped to `-0.0`
as in JavaScript. You can get the desired effect by building simdjson with cmake setting the
`SIMDJSON_MINUS_ZERO_AS_FLOAT` to on: `cmake -B build -D SIMDJSON_MINUS_ZERO_AS_FLOAT=ON`.
You must check the type before accessing the value: it is an error to call `get_int64()` when `number.get_number_type()` is not `number_type::signed_integer` and when `number.is_int64()` is false. You are responsible for this check as the user of the library.
The `get_number()` function is designed with performance in mind. When calling `get_number()`, you scan the number string only once, determining efficiently the type and storing it in an efficient manner.
@@ -2573,6 +2611,20 @@ can use it with features such as `std::optional`:
// value was populated with "3.1416"
```
You can generally convert any answer that would return an `std::string_view`.
```cpp
auto json = R"({"\u0062\u0065\u0062\u0065": 2} })"_padded;
ondemand::parser parser;
ondemand::document doc = parser.iterate(json);
ondemand::object object = doc.get_object();
for (auto field : object) {
std::string key;
error = field.unescaped_key().get(key);
if(error) { /* */ }
}
```
You should be mindful of the trade-off: allocating multiple
`std::string` instances can become expensive.
+25 -1
View File
@@ -7,6 +7,7 @@ An overview of what you need to know to use simdjson, with examples.
* [The Basics: Loading and Parsing JSON Documents](#the-basics-loading-and-parsing-json-documents-using-the-dom-front-end)
* [Using the Parsed JSON](#using-the-parsed-json)
* [C++17 Support](#c17-support)
* [C++20 Support](#c20-support)
* [JSON Pointer](#json-pointer)
* [JSONPath](#jsonpath)
* [Error Handling](#error-handling)
@@ -90,7 +91,7 @@ During the`load` or `parse` calls, neither the input file nor the input string a
For best performance, a `parser` instance should be reused over several files: otherwise you will needlessly reallocate memory, an expensive process. It is also possible to avoid entirely memory allocations during parsing when using simdjson. [See our performance notes for details](performance.md).
If you need a lower-level interface, you may call the function `parser.parse(const char * p, size_t l)` on a pointer `p` while specifying the
length of your input `l` in bytes. To see how to get the very best performance from a low-level approach, you way want to read our [performance notes](https://github.com/simdjson/simdjson/blob/master/doc/performance.md#padding-and-temporary-copies) on this topic (see the Padding and Temporary Copies section).
length of your input `l` in bytes.
*Windows-specific*: Windows users who need to read files with
non-ANSI characters in the name should set their code page to
@@ -119,6 +120,12 @@ Once you have an element, you can navigate it with idiomatic C++ iterators, oper
std::cout << "I parsed " << value << " from " << numberstring.data() << std::endl;
```
The strings contain unescaped valid UTF-8 strings: no unmatched surrogate is allowed.
Internally, numbers are stored as either 64-bit integers or 64-bit floating-point numbers.
Thus it is possible to get the full 64-bit integer range (either signed or unsigned).
By default, the string `-0` is parsed as the integer 0 as in Pytho or C++. If you set the macro
`SIMDJSON_MINUS_ZERO_AS_FLOAT` to `1` when building simdjson, you can get that `-0` is mapped to `-0.0`
as in JavaScript. You can get the desired effect by building simdjson with cmake setting the
`SIMDJSON_MINUS_ZERO_AS_FLOAT` to on: `cmake -B build -D SIMDJSON_MINUS_ZERO_AS_FLOAT=ON`.
* **Field Access:** To get the value of the "foo" field in an object, use `object["foo"]`.
* **Array Iteration:** To iterate through an array, use `for (auto value : array) { ... }`. If you
know the type of the value, you can cast it right there, too! `for (double value : array) { ... }`
@@ -232,6 +239,23 @@ for (dom::key_value_pair field : object) {
}
```
C++20 Support
------------
simdjson library also supports some C++20 feature including `std::ranges`:
```c++
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
] )"_padded;
dom::parser parser;
auto justmodel = [](auto car) { return car["model"]; };
for (auto car : parser.parse(cars_json).get_array() | std::views::transform(justmodel)) {
std::cout << car << std::endl;
}
```
JSON Pointer
------------
+20 -9
View File
@@ -4,9 +4,9 @@ Performance Notes
simdjson strives to be at its fastest *without tuning*, and generally achieves this. However, there
are still some scenarios where tuning can enhance performance.
Once your code is tested, we
further encourage you to define `NDEBUG` in your Release builds to disable additional runtime
further encourage you to define `NDEBUG` in your release builds to disable additional runtime
testing and get the best performance.
* [NDEBUG directive](#ndebug-directive)
* [NDEBUG macro](#ndebug-macro)
* [Reusing the parser for maximum efficiency](#reusing-the-parser-for-maximum-efficiency)
* [Reusing string buffers](#reusing-string-buffers)
* [Server Loops: Long-Running Processes and Memory Capacity](#server-loops-long-running-processes-and-memory-capacity)
@@ -17,16 +17,27 @@ testing and get the best performance.
* [Free Padding](#free-padding)
NDEBUG directive
NDEBUG macro
-------------
In C/C++, the `NDEBUG` pre-processor directive is not set by default. When it is not set, the simdjson library does
many additional checks that may impact negatively the performance. We recommend that, once your code
is well tested, you define `NDEBUG` directive in your Release builds. The `NDEBUG` directive should be defined
prior to including the `simdjson.h` header.
We recommend that you set `NDEBUG` macro in your release builds.
The `NDEBUG` directive is generally independent from optimization flags. For example, setting `-O3` under
GCC does not set the `NDEBUG` directive.
In C/C++, the `NDEBUG` macro is not set by default.
When it is not set, the software may do many additional checks that may impact
negatively the performance. We recommend that, once your code
is well tested, you define `NDEBUG` directive in your release builds.
The `NDEBUG` directive is generally independent from optimization flags.
For example, setting `-O3` under GCC does not set the `NDEBUG` directive.
However, tools like `CMake` automatically
set `NDEBUG` for release builds.
In the simdjson library, we check the `NDEBUG` macro as well as other
macros to make performant release builds. However, the C++ standard
does not provide a definitive approach to determine whether you are
compiling for a release build. Thus we recommend that you follow
the practice of setting the `NDEBUG` macro in release mode to make sure
that you do not get undesirable expensive checks.
Reusing the parser for maximum efficiency
-----------------------------------------
+15 -3
View File
@@ -23,7 +23,7 @@ double from_chars(const char *first, const char* end) noexcept;
}
#ifndef SIMDJSON_EXCEPTIONS
#if __cpp_exceptions
#if defined(__cpp_exceptions) || defined(_CPPUNWIND)
#define SIMDJSON_EXCEPTIONS 1
#else
#define SIMDJSON_EXCEPTIONS 0
@@ -226,12 +226,14 @@ double from_chars(const char *first, const char* end) noexcept;
// even if we do not have C++17 support.
#ifdef __cpp_lib_string_view
#define SIMDJSON_HAS_STRING_VIEW
#include <string_view>
#endif
// Some systems have string_view even if we do not have C++17 support,
// and even if __cpp_lib_string_view is undefined, it is the case
// with Apple clang version 11.
// We must handle it. *This is important.*
#ifndef _MSC_VER
#ifndef SIMDJSON_HAS_STRING_VIEW
#if defined __has_include
// do not combine the next #if with the previous one (unsafe)
@@ -247,6 +249,7 @@ double from_chars(const char *first, const char* end) noexcept;
#endif // __has_include (<string_view>)
#endif // defined __has_include
#endif // def SIMDJSON_HAS_STRING_VIEW
#endif // def _MSC_VER
// end of complicated but important routine to try to detect string_view.
//
@@ -279,16 +282,25 @@ namespace std {
// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer
// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to
// set the __OPTIMIZE__ macro).
// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS
// is not defined, irrespective of the compiler.
// We recommend that users set NDEBUG in release builds, so that
// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default,
// irrespective of the compiler.
#ifndef SIMDJSON_DEVELOPMENT_CHECKS
#ifdef _MSC_VER
// Visual Studio seems to set _DEBUG for debug builds.
#ifdef _DEBUG
// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined
// and NDEBUG is not defined.
#if defined(_DEBUG) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // _DEBUG
#else // _MSC_VER
// All other compilers appear to set __OPTIMIZE__ to a positive integer
// when the compiler is optimizing.
#ifndef __OPTIMIZE__
// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__
// and NDEBUG are not defined.
#if !defined(__OPTIMIZE__) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // __OPTIMIZE__
#endif // _MSC_VER
+2 -2
View File
@@ -57,7 +57,7 @@ public:
iterator& operator=(const iterator&) noexcept = default;
private:
simdjson_inline iterator(const internal::tape_ref &tape) noexcept;
internal::tape_ref tape;
internal::tape_ref tape{};
friend class array;
};
@@ -148,7 +148,7 @@ public:
private:
simdjson_inline array(const internal::tape_ref &tape) noexcept;
internal::tape_ref tape;
internal::tape_ref tape{};
friend class element;
friend struct simdjson_result<element>;
template<typename T>
+1 -1
View File
@@ -492,7 +492,7 @@ public:
private:
simdjson_inline element(const internal::tape_ref &tape) noexcept;
internal::tape_ref tape;
internal::tape_ref tape{};
friend class document;
friend class object;
friend class array;
+2 -2
View File
@@ -89,7 +89,7 @@ public:
private:
simdjson_inline iterator(const internal::tape_ref &tape) noexcept;
internal::tape_ref tape;
internal::tape_ref tape{};
friend class object;
};
@@ -224,7 +224,7 @@ public:
private:
simdjson_inline object(const internal::tape_ref &tape) noexcept;
internal::tape_ref tape;
internal::tape_ref tape{};
friend class element;
friend struct simdjson_result<element>;
+17
View File
@@ -132,6 +132,23 @@ simdjson_warn_unused simdjson_inline error_code simdjson_result<T>::get(T &value
return std::forward<internal::simdjson_result_base<T>>(*this).get(value);
}
template<typename T>
simdjson_warn_unused simdjson_inline error_code
simdjson_result<T>::get(std::string &value) && noexcept
#if SIMDJSON_SUPPORTS_DESERIALIZATION
requires (!std::is_same_v<T, std::string>)
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
{
// SFINAE : n'active que pour T = std::string_view
static_assert(std::is_same<T, std::string_view>::value, "simdjson_result<T>::get(std::string&) n'est disponible que pour T = std::string_view");
std::string_view v;
error_code error = std::forward<simdjson_result<T>>(*this).get(v);
if (!error) {
value.assign(v.data(), v.size());
}
return error;
}
template<typename T>
simdjson_inline error_code simdjson_result<T>::error() const noexcept {
return internal::simdjson_result_base<T>::error();
+12 -1
View File
@@ -196,6 +196,7 @@ struct simdjson_result_base : protected std::pair<T, error_code> {
* @throw simdjson_error if there was an error.
*/
simdjson_inline operator T&&() && noexcept(false);
#endif // SIMDJSON_EXCEPTIONS
/**
@@ -252,7 +253,17 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
* @param value The variable to assign the value to. May not be set if there is an error.
*/
simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept;
//
/**
* Copy the value to a provided std::string, only enabled for std::string_view.
*
* @param value The variable to assign the value to. May not be set if there is an error.
*/
simdjson_warn_unused simdjson_inline error_code get(std::string &value) && noexcept
#if SIMDJSON_SUPPORTS_DESERIALIZATION
requires (!std::is_same_v<T, std::string>)
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
;
/**
* The error.
*/
+16 -1
View File
@@ -647,7 +647,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -1108,6 +1117,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -45,4 +45,8 @@
#include "simdjson/generic/ondemand/token_iterator-inl.h"
#include "simdjson/generic/ondemand/value_iterator-inl.h"
// JSON builder, ideally they should not be part of the ondemand directory
// but it is convenient for now to have them here.
#include "simdjson/generic/ondemand/json_string_builder.h"
#include "simdjson/generic/ondemand/json_string_builder-inl.h"
@@ -0,0 +1,54 @@
/**
* This file is part of the builder API. It is temporarily in the ondemand directory
* but we will move it to a builder directory later.
*/
#ifndef SIMDJSON_GENERIC_BUILDER_INL_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_BUILDER_INL_H
#include "simdjson/generic/builder/json_string_builder.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace builder {
simdjson_inline string_builder::string_builder(size_t initial_capacity) :
buffer(new (std::nothrow) char[initial_capacity]),
position(0), capacity(buffer.get() != nullptr ? initial_capacity : 0),
is_valid(buffer.get() != nullptr) {}
/***
char *padded_buffer = new (std::nothrow) char[totalpaddedlength];
if (padded_buffer == nullptr) {
return nullptr;
}
*/
simdjson_inline bool string_builder::capacity_check(size_t upcoming_bytes) {
if (upcoming_bytes <= capacity - position) { return true; }
// check for overflow:
if (position + upcoming_bytes < position) { return false; }
grow_buffer(std::max(capacity * 2, position + upcoming_bytes));
return is_valid;
}
simdjson_inline void string_builder::grow_buffer(size_t desired_capacity) {
if (!is_valid) { return; }
std::unique_ptr<char[]> new_buffer(new (std::nothrow) char[desired_capacity]);
if (new_buffer.get() == nullptr) {
is_valid = false;
capacity = 0;
return;
}
memcpy(new_buffer.get(), buffer.get(), position);
buffer.swap(new_buffer);
capacity = desired_capacity;
}
simdjson_inline size_t string_builder::size() const {
return is_valid ? position : 0;
}
} // namespace builder
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_GENERIC_BUILDER_INL_H
@@ -0,0 +1,146 @@
/**
* This file is part of the builder API. It is temporarily in the ondemand directory
* but we will move it to a builder directory later.
*/
#ifndef SIMDJSON_GENERIC_BUILDER_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_BUILDER_H
#include "simdjson/generic/implementation_simdjson_result_base.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace builder {
//////////////////////////
// TODO: at the end of the processing, possibly as an optional step, we should
// validate the UTF-8 of the string.
/////////////////////////
/**
* A builder for JSON strings representing documents. This is a low-level
* builder that is not meant to be used directly by end-users. Though it
* supports atomic types (Booleans, strings), it does not support composed
* types (arrays and objects).
*
* Ultimately, this class should support kernel-specific optimizations. E.g.,
* it may make use of SIMD instructions to escape strings faster.
*/
class string_builder {
public:
simdjson_inline string_builder(size_t initial_capacity);
/**
* Append number (includes Booleans). Booleans are mapped to the strings
* false and true. Numbers are converted to strings abiding by the JSON standard.
* Floating-point numbers are converted to the shortest string that 'correctly'
* represents the number.
*/
template<typename number_type,
typename = typename std::enable_if<std::is_arithmetic<number_type>::value>::type>
simdjson_inline void append(number_type v) noexcept;
/**
* Append character c.
*/
simdjson_inline void append(char c) noexcept;
/**
* Append the string 'null'.
*/
simdjson_inline void append_null() noexcept;
/**
* Clear the content.
*/
simdjson_inline void clear() noexcept;
/**
* Append the std::string_view, after escaping it.
* There is no UTF-8 validation.
*/
simdjson_inline void escape_and_append(std::string_view input) noexcept;
/**
* Append the std::string_view surrounded by double quotes, after escaping it.
* There is no UTF-8 validation.
*/
simdjson_inline void escape_and_append_with_quotes(std::string_view input) noexcept;
/**
* Append the C string directly, without escaping.
* There is no UTF-8 validation.
*/
simdjson_inline void append_raw(const char *c) noexcept;
/**
* Append the std::string_view directly, without escaping.
* There is no UTF-8 validation.
*/
simdjson_inline void append_raw(std::string_view str) noexcept;
/**
* Append len characters from str.
* There is no UTF-8 validation.
*/
simdjson_inline void append_raw(const char *str, size_t len) noexcept;
#if SIMDJSON_EXCEPTIONS
/**
* Creates an std::string from the written JSON buffer.
* Throws if memory allocation failed
*/
simdjson_inline operator std::string() const noexcept(false);
/**
* Creates an std::string_view from the written JSON buffer.
* Throws if memory allocation failed
*/
simdjson_inline operator std::string_view() const noexcept(false);
#endif
/**
* Returns a view on the written JSON buffer. Returns an error
* if memory allocation failed.
*/
simdjson_inline simdjson_result<std::string_view> view() const noexcept;
/**
* Appends the null character to the buffer and returns
* a pointer to the beginning of the written JSON buffer.
* Returns an error if memory allocation failed.
*/
simdjson_inline simdjson_result<const char *> c_str();
/**
* Returns the current size of the written JSON buffer.
* If an error occurred, returns 0.
*/
simdjson_inline size_t size() const;
private:
/**
* Returns true if we can write at least upcoming_bytes bytes.
* The underlying buffer is reallocated if needed. It is designed
* to be called before writing to the buffer. It should be fast.
*/
simdjson_inline bool capacity_check(size_t upcoming_bytes);
/**
* Grow the buffer to at least desired_capacity bytes.
* If the allocation fails, is_valid is set to false. We expect
* that this function would not be repeatedly called.
*/
simdjson_inline void grow_buffer(size_t desired_capacity);
std::unique_ptr<char[]> buffer;
size_t position;
size_t capacity;
bool is_valid{true};
};
}
}
} // namespace simdjson
#endif // SIMDJSON_GENERIC_BUILDER_H
+1 -1
View File
@@ -2,7 +2,7 @@
#define SIMDJSON_JSONPATHUTIL_H
#include <string>
#include <string_view>
#include "simdjson/common_defs.h"
namespace simdjson {
/**
+1
View File
@@ -191,6 +191,7 @@ using std::size_t;
#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
// then do away with asserts and use __assume.
// We still recommend that our users set NDEBUG in release mode.
#if SIMDJSON_VISUAL_STUDIO
#define SIMDJSON_UNREACHABLE() __assume(0)
#define SIMDJSON_ASSUME(COND) __assume(COND)
+3 -3
View File
@@ -4,7 +4,7 @@
#define SIMDJSON_SIMDJSON_VERSION_H
/** The version of simdjson being used (major.minor.revision) */
#define SIMDJSON_VERSION "3.12.3"
#define SIMDJSON_VERSION "3.13.0"
namespace simdjson {
enum {
@@ -15,11 +15,11 @@ enum {
/**
* The minor version (major.MINOR.revision) of simdjson being used.
*/
SIMDJSON_VERSION_MINOR = 12,
SIMDJSON_VERSION_MINOR = 13,
/**
* The revision (major.minor.REVISION) of simdjson being used.
*/
SIMDJSON_VERSION_REVISION = 3
SIMDJSON_VERSION_REVISION = 0
};
} // namespace simdjson
+1 -1
View File
@@ -52,7 +52,7 @@ if (Python3_Interpreter_FOUND AND (NOT WIN32))
# file regeneration in the source directory without the user's knowledge.
# OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/simdjson.cpp ${CMAKE_CURRENT_SOURCE_DIR}/simdjson.h ${CMAKE_CURRENT_SOURCE_DIR}/amalgamate_demo.cpp ${CMAKE_CURRENT_SOURCE_DIR}/README.md
COMMAND ${CMAKE_COMMAND} -E copy ${SINGLEHEADER_FILES} ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS ${SINGLEHEADER_FILES}
POST_BUILD ${SINGLEHEADER_FILES}
)
endif()
+63 -16
View File
@@ -10,12 +10,41 @@ import os
import re
import shutil
import datetime
import json
from typing import Dict, List, Optional, Set, TextIO, Union, cast
# Check for Python 3, this does not actually work.
if sys.version_info < (3, 0):
sys.stdout.write("Sorry, requires Python 3.x or better\n")
sys.exit(1)
rules = """
We refer your to the HACKING.md file for more information on how the project is organized.
To help understand the error, here are the rules for including files in simdjson:
All implementation-specific files, including arm64.h, arm64/implementation.h and
arm64/ondemand.h, must be within SIMDJSON_CONDITIONAL_INCLUDE blocks.
Top-level headers must not be included in any SIMDJSON_CONDITIONAL_INCLUDE block.
Generic files must be included only in amalgamator files (arm64.h,
arm64/implementation.h, arm64/ondemand.h, generic/amalgamated.h).
We fail if an implementation-specific file is included more than once in the same block.
We fail if a generic file is included more than once per implementation in the same block.
Tip: generally, "file" will search the including file's source directory first, then
the search paths while <file> does it the other way around.
We prefer to use <> in simdjson headers to avoid accidentally including a file from the
wrong directory.
The amalgamate.py script checks that all files are included.
"""
SCRIPTPATH = os.path.dirname(os.path.abspath(sys.argv[0]))
PROJECTPATH = os.path.dirname(SCRIPTPATH)
print(f"SCRIPTPATH={SCRIPTPATH} PROJECTPATH={PROJECTPATH}")
@@ -62,6 +91,22 @@ class SimdjsonFile:
def __str__(self):
return self.include_path
def dump(self):
return {
'root': self.root,
'include_path': self.include_path,
'includes': [include.include_path for include in self.includes],
'included_from': [included_from.include_path for included_from in self.included_from],
'editor_only_includes': [editor_only_include.include_path for editor_only_include in self.editor_only_includes],
'editor_only_included_from': [editor_only_included_from.include_path for editor_only_included_from in self.editor_only_included_from],
'processed': self.processed,
'dependency_file': self.dependency_file.include_path if self.dependency_file else None,
'is_amalgamator': self.is_amalgamator,
'implementation': self.implementation,
}
def json(self):
return json.dumps(self.dump(), indent=4, sort_keys=True, ensure_ascii=False)
def __repr__(self):
return self.include_path
@@ -162,20 +207,21 @@ class SimdjsonFile:
def add_include(self, include: 'SimdjsonFile'):
if self.is_conditional_include:
assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
# If I have a dependency file, I can only include something that has a dependency file.
assert include.is_conditional_include, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
# TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
# if include.amalgamator_file:
# assert include.amalgamator_file == self, f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
else:
assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file."
assert include.is_amalgamator or not include.is_conditional_include, f"{self} cannot include {include} because it is an amalgamated file. {rules}"
self.includes.append(include)
include.included_from.add(self)
def add_editor_only_include(self, include: 'SimdjsonFile'):
assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file."
assert self.is_conditional_include, f"Cannot use #ifndef SIMDJSON_CONDITIONAL_INCLUDE in {self} because it is not an amalgamated file. {rules}"
if not include.is_conditional_include:
assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE."
assert self.dependency_file, f"{self} cannot include {include} without #ifndef SIMDJSON_CONDITIONAL_INCLUDE. {rules}"
# TODO make sure we only include amalgamated files that are guaranteed to be included with us (or before us)
# elif include.amalgamator_file:
# assert self.is_amalgamated_before(self.amalgamator_file), f"{self} cannot include {include}: it should be included from {include.amalgamator_file} instead."
@@ -190,11 +236,11 @@ class SimdjsonFile:
if file.dependency_file == self:
for editor_only_include in file.editor_only_includes:
if not editor_only_include.is_conditional_include:
assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}."
assert editor_only_include in self.includes, f"{file} includes {editor_only_include}, but it is not included from {self}. It must be added to {self}. {rules}"
if editor_only_include in extra_include_set:
extra_include_set.remove(editor_only_include)
assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files."
assert len(extra_include_set) == 0, f"{self} unnecessarily includes {extra_include_set}. They are not included in the corresponding amalgamated files. {rules}"
class SimdjsonRepository:
def __init__(self, project_path: str, relative_roots: List[RelativeRoot]):
@@ -320,6 +366,7 @@ class Amalgamator:
assert not self.editor_only_region
with open(file.absolute_path, 'r') as fid2:
print(f"including: {file}")
for line in fid2:
line = line.rstrip('\n')
@@ -329,9 +376,9 @@ class Amalgamator:
# Ignore lines inside #ifndef SIMDJSON_CONDITIONAL_INCLUDE
if re.search(r'^#ifndef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file!"
assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack}"
assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row"
assert file.is_conditional_include, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE but is not an amalgamated file! {rules}"
assert self.in_conditional_include_block, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE without a prior #define SIMDJSON_CONDITIONAL_INCLUDE: {self.include_stack} {rules}"
assert not self.editor_only_region, f"{file} uses #ifndef SIMDJSON_CONDITIONAL_INCLUDE twice in a row {rules}"
self.editor_only_region = True
# Handle ignored lines (and ending ignore blocks)
@@ -349,7 +396,7 @@ class Amalgamator:
self.editor_only_region = False
continue
assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE"
assert not end_ignore, f"{file} has #endif // SIMDJSON_CONDITIONAL_INCLUDE without #ifndef SIMDJSON_CONDITIONAL_INCLUDE {rules}"
# Handle #include lines
included = re.search(r'^#include\s+["<]([^">]*)[">]', line)
@@ -376,26 +423,26 @@ class Amalgamator:
self.implementation = None
elif re.search(r'\bSIMDJSON_IMPLEMENTATION\b', line) and file.include_path != IMPLEMENTATION_DETECTION_H:
# copy the line, with SIMDJSON_IMPLEMENTATION replace to what it is currently defined to
assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}"
assert self.implementation, f"Use of SIMDJSON_IMPLEMENTATION while not defined in {file}: {line}\n{rules}"
line = re.sub(r'\bSIMDJSON_IMPLEMENTATION\b',self.implementation,line)
# Handle defining and undefining SIMDJSON_CONDITIONAL_INCLUDE
defined = re.search(r'^#define\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line)
if defined:
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed."
assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE"
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE defined in amalgamated file {file}! Not allowed. {rules}"
assert not self.in_conditional_include_block, f"{file} redefines SIMDJSON_CONDITIONAL_INCLUDE {rules}"
self.in_conditional_include_block = True
self.found_includes_per_conditional_block.clear()
self.write(f'/* defining SIMDJSON_CONDITIONAL_INCLUDE */')
elif re.search(r'^#undef\s+SIMDJSON_CONDITIONAL_INCLUDE\s*$', line):
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed."
assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it"
assert not file.is_conditional_include, "SIMDJSON_CONDITIONAL_INCLUDE undefined in amalgamated file {file}! Not allowed. {rules}"
assert self.in_conditional_include_block, f"{file} undefines SIMDJSON_CONDITIONAL_INCLUDE without defining it {rules}"
self.write(f'/* undefining SIMDJSON_CONDITIONAL_INCLUDE */')
self.in_conditional_include_block = False
self.write(line)
assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE"
assert not self.editor_only_region, f"{file} ended without #endif // SIMDJSON_CONDITIONAL_INCLUDE {rules}"
self.write(f"/* end file {self.file_to_str(file)} */")
+174 -13
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2025-03-27 15:01:10 -0400. Do not edit! */
/* auto-generated on 2025-07-14 11:45:25 -0400. Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
@@ -298,6 +298,7 @@ using std::size_t;
#if defined(NDEBUG) || defined(__OPTIMIZE__) || (defined(_MSC_VER) && !defined(_DEBUG))
// If NDEBUG is set, or __OPTIMIZE__ is set, or we are under MSVC in release mode,
// then do away with asserts and use __assume.
// We still recommend that our users set NDEBUG in release mode.
#if SIMDJSON_VISUAL_STUDIO
#define SIMDJSON_UNREACHABLE() __assume(0)
#define SIMDJSON_ASSUME(COND) __assume(COND)
@@ -373,7 +374,7 @@ double from_chars(const char *first, const char* end) noexcept;
}
#ifndef SIMDJSON_EXCEPTIONS
#if __cpp_exceptions
#if defined(__cpp_exceptions) || defined(_CPPUNWIND)
#define SIMDJSON_EXCEPTIONS 1
#else
#define SIMDJSON_EXCEPTIONS 0
@@ -576,12 +577,14 @@ double from_chars(const char *first, const char* end) noexcept;
// even if we do not have C++17 support.
#ifdef __cpp_lib_string_view
#define SIMDJSON_HAS_STRING_VIEW
#include <string_view>
#endif
// Some systems have string_view even if we do not have C++17 support,
// and even if __cpp_lib_string_view is undefined, it is the case
// with Apple clang version 11.
// We must handle it. *This is important.*
#ifndef _MSC_VER
#ifndef SIMDJSON_HAS_STRING_VIEW
#if defined __has_include
// do not combine the next #if with the previous one (unsafe)
@@ -597,6 +600,7 @@ double from_chars(const char *first, const char* end) noexcept;
#endif // __has_include (<string_view>)
#endif // defined __has_include
#endif // def SIMDJSON_HAS_STRING_VIEW
#endif // def _MSC_VER
// end of complicated but important routine to try to detect string_view.
//
@@ -2339,16 +2343,25 @@ namespace std {
// It could also wrongly set SIMDJSON_DEVELOPMENT_CHECKS (e.g., if the programmer
// sets _DEBUG in a release build under Visual Studio, or if some compiler fails to
// set the __OPTIMIZE__ macro).
// We make it so that if NDEBUG is defined, then SIMDJSON_DEVELOPMENT_CHECKS
// is not defined, irrespective of the compiler.
// We recommend that users set NDEBUG in release builds, so that
// SIMDJSON_DEVELOPMENT_CHECKS is not defined in release builds by default,
// irrespective of the compiler.
#ifndef SIMDJSON_DEVELOPMENT_CHECKS
#ifdef _MSC_VER
// Visual Studio seems to set _DEBUG for debug builds.
#ifdef _DEBUG
// We set SIMDJSON_DEVELOPMENT_CHECKS to 1 if _DEBUG is defined
// and NDEBUG is not defined.
#if defined(_DEBUG) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // _DEBUG
#else // _MSC_VER
// All other compilers appear to set __OPTIMIZE__ to a positive integer
// when the compiler is optimizing.
#ifndef __OPTIMIZE__
// We only set SIMDJSON_DEVELOPMENT_CHECKS if both __OPTIMIZE__
// and NDEBUG are not defined.
#if !defined(__OPTIMIZE__) && !defined(NDEBUG)
#define SIMDJSON_DEVELOPMENT_CHECKS 1
#endif // __OPTIMIZE__
#endif // _MSC_VER
@@ -2607,6 +2620,7 @@ struct simdjson_result_base : protected std::pair<T, error_code> {
* @throw simdjson_error if there was an error.
*/
simdjson_inline operator T&&() && noexcept(false);
#endif // SIMDJSON_EXCEPTIONS
/**
@@ -2663,7 +2677,17 @@ struct simdjson_result : public internal::simdjson_result_base<T> {
* @param value The variable to assign the value to. May not be set if there is an error.
*/
simdjson_warn_unused simdjson_inline error_code get(T &value) && noexcept;
//
/**
* Copy the value to a provided std::string, only enabled for std::string_view.
*
* @param value The variable to assign the value to. May not be set if there is an error.
*/
simdjson_warn_unused simdjson_inline error_code get(std::string &value) && noexcept
#if SIMDJSON_SUPPORTS_DESERIALIZATION
requires (!std::is_same_v<T, std::string>)
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
;
/**
* The error.
*/
@@ -4663,6 +4687,23 @@ simdjson_warn_unused simdjson_inline error_code simdjson_result<T>::get(T &value
return std::forward<internal::simdjson_result_base<T>>(*this).get(value);
}
template<typename T>
simdjson_warn_unused simdjson_inline error_code
simdjson_result<T>::get(std::string &value) && noexcept
#if SIMDJSON_SUPPORTS_DESERIALIZATION
requires (!std::is_same_v<T, std::string>)
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
{
// SFINAE : n'active que pour T = std::string_view
static_assert(std::is_same<T, std::string_view>::value, "simdjson_result<T>::get(std::string&) n'est disponible que pour T = std::string_view");
std::string_view v;
error_code error = std::forward<simdjson_result<T>>(*this).get(v);
if (!error) {
value.assign(v.data(), v.size());
}
return error;
}
template<typename T>
simdjson_inline error_code simdjson_result<T>::error() const noexcept {
return internal::simdjson_result_base<T>::error();
@@ -9726,7 +9767,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -10187,6 +10237,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -16086,7 +16142,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -16547,6 +16612,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -22310,7 +22381,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -22771,6 +22851,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -28690,7 +28776,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -29151,6 +29246,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -35432,7 +35533,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -35893,6 +36003,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -41998,7 +42114,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -42459,6 +42584,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -48009,7 +48140,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -48470,6 +48610,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
@@ -53619,7 +53765,16 @@ simdjson_inline error_code parse_number(const uint8_t *const src, W &writer) {
if (i > uint64_t(INT64_MAX)) {
WRITE_UNSIGNED(i, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(i == 0 && negative) {
// We have to write -0.0 instead of 0
WRITE_DOUBLE(-0.0, src, writer);
} else {
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
}
#else
WRITE_INTEGER(negative ? (~i+1) : i, src, writer);
#endif
}
if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); }
return SUCCESS;
@@ -54080,6 +54235,12 @@ simdjson_unused simdjson_inline simdjson_result<number_type> get_number_type(con
if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) {
return number_type::big_integer;
}
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
if(digit_count == 1 && src[0] == '0') {
// We have to write -0.0 instead of 0
return number_type::floating_point_number;
}
#endif
return number_type::signed_integer;
}
// Let us check if we have a big integer (>=2**64).
+1847 -22
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -13,7 +13,7 @@ add_cpp_test(minify_tests LABELS other acceptance per_implementation)
add_cpp_test(padded_string_tests LABELS other acceptance )
add_cpp_test(prettify_tests LABELS other acceptance per_implementation)
if(MSVC AND BUILD_SHARED_LIBS)
if(WIN32 AND BUILD_SHARED_LIBS)
# Copy the simdjson dll into the tests directory
add_custom_command(TARGET unicode_tests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
+19 -2
View File
@@ -113,8 +113,25 @@ if(NOT (MSVC AND MSVC_VERSION LESS 1920))
target_compile_definitions(readme_examples_will_fail_with_exceptions_off PRIVATE SIMDJSON_EXCEPTIONS=0)
endif()
#
# Test C++20 features
#
if(MSVC AND BUILD_SHARED_LIBS)
# Add the tests if we're on:
# 1. Visual Studio 2022 v17.6 or later
# 2. GCC v14.0.0 or later (GCC v13.0.0 cannot handle pipe oprator of lambda)
# 3. Clang v15.0.0 or later (certain version C++ headers occur error when compiling)
if(
(MSVC AND MSVC_VERSION LESS 1930) OR
(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "14.0.0") OR
(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "15.0.0")
)
message(STATUS "compiler id: ${CMAKE_CXX_COMPILER_ID} version: ${CMAKE_CXX_COMPILER_VERSION}")
add_cpp_test(ranges_test LABELS dom acceptance per_implementation)
set_target_properties(ranges_test PROPERTIES CXX_STANDARD 20 CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF)
endif()
if(WIN32 AND BUILD_SHARED_LIBS)
add_custom_command(TARGET basictests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
@@ -135,4 +152,4 @@ if(NOT MSVC)
if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION})
message(FATAL_ERROR "The library target does not have the proper version information." )
endif()
endif()
endif()
-8
View File
@@ -12,8 +12,6 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS
#include "gason.h"
#include "json11.hpp"
#include "rapidjson/document.h"
#include "rapidjson/reader.h" // you have to check in the submodule
#include "rapidjson/stringbuffer.h"
@@ -122,10 +120,6 @@ int main(int argc, char *argv[]) {
return EXIT_SUCCESS;
}
bool rapid_correct = (d.Parse((const char *)buffer).HasParseError() == false);
std::string json11err;
bool dropbox_correct = ((json11::Json::parse(buffer, json11err).is_null()) ||
(!json11err.empty())) == false;
bool fastjson_correct = fastjson_parse(buffer);
JsonValue value;
JsonAllocator allocator;
@@ -174,8 +168,6 @@ int main(int argc, char *argv[]) {
rapid_correct_checkencoding ? "correct" : "invalid");
printf("sajson : %s \n",
sajson_correct ? "correct" : "invalid");
printf("dropbox : %s \n",
dropbox_correct ? "correct" : "invalid");
printf("fastjson : %s \n",
fastjson_correct ? "correct" : "invalid");
printf("gason : %s \n",
+31 -2
View File
@@ -399,7 +399,11 @@ namespace number_tests {
bool specific_tests() {
std::cout << __func__ << std::endl;
return basic_test_64bit("-1e-999", -0.0) &&
return
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
basic_test_64bit("-0", -0.0) &&
#endif // SIMDJSON_MINUS_ZERO_AS_FLOAT
basic_test_64bit("-1e-999", -0.0) &&
basic_test_64bit("-2402844368454405395.2",-2402844368454405395.2) &&
basic_test_64bit("4503599627370496.5", 4503599627370496.5) &&
basic_test_64bit("4503599627475352.5", 4503599627475352.5) &&
@@ -458,6 +462,31 @@ namespace parse_api_tests {
return true;
}
#if SIMDJSON_EXCEPTIONS
bool issue_2375() {
TEST_START();
std::string jsonData =
R"eos({"asset":{"version":"1.0"},"geometricError":100,"root":{"boundingVolume":{"region":[10,0,10,10,0,109]},"geometricError":100,"refine":"ADD","children":[{"boundingVolume":{"region":[20,0,20,0,0,20]},"geometricError":70,"content":{"url":"city/tileset.json"}},{"transform":[4,1,0,0,-0.7,3.1,3.8,0,0.9,-3.7,3.21,0,12,-47,40,1],"boundingVolume":{"region":[-1.3,0.6,-1.39,0.6,0,20]},"geometricError":0,"content":{"url":"building.b3dm"}},{"transform":[0.9,0.2,0,0,-0.15,0.62,0.76,0,0.19,-0.74,0.64,0,12,-47,40,1],"viewerRequestVolume":{"region":[-1.3,0.6,-1.31,0.6,0,20]},"boundingVolume":{"region":[-1.31,0.6,-1.3,0.6,0,20]},"geometricError":0,"content":{"url":"points.pnts"}}]}})eos";
simdjson::dom::parser parser;
auto json = parser.parse(jsonData.data(), jsonData.size());
const simdjson::dom::element jsonElement = json.value();
const simdjson::dom::element rootElement = jsonElement["root"];
if (jsonElement["asset"]["gltfUpAxis"].is_string()) {
if (jsonElement["asset"]["gltfUpAxis"].get_string().value_unsafe() == std::string_view("Z")) {
printf("up\n");
}
}
if (rootElement.is_object()) {
printf("I can confirm that root is an object!\n");
return true;
}
printf("root is not an object!\n");
return false;
}
bool issue679() {
std::cout << "Running " << __func__ << std::endl;
auto input = "[1, 2, 3]"_padded;
@@ -742,6 +771,7 @@ namespace parse_api_tests {
parser_load_exception() &&
parser_load_many_exception() &&
issue679() &&
issue_2375() &&
#endif
true;
}
@@ -1245,7 +1275,6 @@ namespace dom_api_tests {
bool numeric_values_exception() {
std::cout << "Running " << __func__ << std::endl;
dom::parser parser;
ASSERT_EQUAL( uint64_t(parser.parse("0"_padded)), 0);
ASSERT_EQUAL( int64_t(parser.parse("0"_padded)), 0);
ASSERT_EQUAL( double(parser.parse("0"_padded)), 0);
+59
View File
@@ -0,0 +1,59 @@
#include <iostream>
#include <ranges>
#include <string_view>
#include "simdjson.h"
#include "test_macros.h"
#include "test_main.h"
using namespace simdjson;
namespace ranges_test {
bool printout() {
TEST_START();
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
] )"_padded;
dom::parser parser;
auto justmodel = [](auto car) { return car["model"]; };
for (auto car :
parser.parse(cars_json).get_array() | std::views::transform(justmodel)) {
std::cout << car << std::endl;
}
TEST_SUCCEED();
}
bool checkvalues() {
TEST_START();
auto cars_json = R"( [
{ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] },
{ "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] }
] )"_padded;
dom::parser parser;
auto justmodel = [](auto car) -> std::string_view {
return car["model"].get_string();
};
auto cars =
parser.parse(cars_json).get_array() | std::views::transform(justmodel);
const std::vector<std::string_view> expected{"Camry", "Soul", "Tercel"};
ASSERT_TRUE(std::ranges::equal(cars, expected));
TEST_SUCCEED();
}
bool run() {
return
#if SIMDJSON_EXCEPTIONS
printout() && checkvalues() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace ranges_test
int main(int argc, char *argv[]) {
return test_main(argc, argv, ranges_test::run);
}
+2 -2
View File
@@ -42,9 +42,9 @@ if(HAVE_POSIX_FORK AND HAVE_POSIX_WAIT) # assert tests use fork and wait, which
endif()
# Copy the simdjson dll into the tests directory
if(MSVC AND BUILD_SHARED_LIBS)
if(WIN32 AND BUILD_SHARED_LIBS)
add_custom_command(TARGET ondemand_parse_api_tests POST_BUILD # Adds a post-build event
COMMAND ${CMAKE_COMMAND} -E copy_if_different # which executes "cmake -E copy_if_different..."
"$<TARGET_FILE:simdjson>" # <--this is in-file
"$<TARGET_FILE_DIR:ondemand_parse_api_tests>") # <--this is out-file path
endif(MSVC AND BUILD_SHARED_LIBS)
endif(WIN32 AND BUILD_SHARED_LIBS)
@@ -83,6 +83,31 @@ bool custom_test() {
TEST_SUCCEED();
}
bool uint8_t_test() {
TEST_START();
simdjson::ondemand::parser parser;
const simdjson::padded_string json =
R"({"data" : [1,2,3,4]})"_padded;
simdjson::ondemand::document d = parser.iterate(json);
std::vector<uint8_t> array = d["data"].get<std::vector<uint8_t>>();
std::vector<uint8_t> expected = {1, 2, 3, 4};
ASSERT_EQUAL(array, expected);
TEST_SUCCEED();
}
bool append_test() {
TEST_START();
simdjson::ondemand::parser parser;
const simdjson::padded_string json =
R"({"data" : [1,2,3,4]})"_padded;
simdjson::ondemand::document d = parser.iterate(json);
std::vector<uint32_t> array = {0, 0};
d["data"].get<std::vector<uint32_t>>(array);
std::vector<uint32_t> expected = {0, 0, 1, 2, 3, 4};
ASSERT_EQUAL(array, expected);
TEST_SUCCEED();
}
bool readme_test() {
TEST_START();
auto const json = R"( { "make": "Toyota", "model": "Camry", "year": 2018,
@@ -162,6 +187,8 @@ bool simple_document_test_no_except() {
bool run() {
return
#if SIMDJSON_EXCEPTIONS && defined(__cpp_concepts)
append_test() &&
uint8_t_test() &&
readme_test() &&
custom_test() &&
simple_document_test() &&
+26 -2
View File
@@ -312,7 +312,30 @@ namespace number_tests {
ASSERT_EQUAL(number.get_number_type(), ondemand::number_type::floating_point_number);
ASSERT_EQUAL(number.get_double(), 1e9);
TEST_SUCCEED();
}
}
bool minus_zero() {
TEST_START();
ondemand::parser parser;
auto json = "-0"_padded;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
ondemand::number number;
ASSERT_SUCCESS(doc.get_number().get(number));
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
ASSERT_EQUAL(number.get_number_type(), ondemand::number_type::floating_point_number);
#else
ASSERT_EQUAL(number.get_number_type(), ondemand::number_type::signed_integer);
#endif
ondemand::number_type nt{};
ASSERT_SUCCESS(doc.get_number_type().get(nt));
#if SIMDJSON_MINUS_ZERO_AS_FLOAT
ASSERT_EQUAL(nt, ondemand::number_type::floating_point_number);
#else
ASSERT_EQUAL(nt, ondemand::number_type::signed_integer);
#endif
TEST_SUCCEED();
}
bool issue1878() {
TEST_START();
@@ -504,7 +527,8 @@ namespace number_tests {
}
bool run() {
return gigantic_big_int() &&
return minus_zero() &&
gigantic_big_int() &&
big_int_not_zero() &&
negative_big_int() &&
issue2099() &&
+34 -1
View File
@@ -23,6 +23,24 @@ namespace object_tests {
TEST_SUCCEED();
}
bool testing_to_string() {
TEST_START();
auto json = R"({"\u0062\u0065\u0062\u0065": 2} })"_padded;
ondemand::parser parser;
ondemand::document doc;
ASSERT_SUCCESS(parser.iterate(json).get(doc));
ondemand::object object;
ASSERT_SUCCESS(doc.get_object().get(object));
for (auto field : object) {
std::string key;
ASSERT_SUCCESS(field.unescaped_key().get(key));
ASSERT_EQUAL(key, "bebe");
}
TEST_SUCCEED();
}
bool issue1977() {
TEST_START();
auto json = R"({"1": 2} foo })"_padded;
@@ -258,6 +276,19 @@ namespace object_tests {
#if SIMDJSON_EXCEPTIONS
bool testing_to_string_exception() {
TEST_START();
auto json = R"({"\u0062\u0065\u0062\u0065": 2} })"_padded;
ondemand::parser parser;
ondemand::document doc = parser.iterate(json);
ondemand::object object = doc.get_object();
for (auto field : object) {
std::string key;
ASSERT_SUCCESS(field.unescaped_key().get(key));
}
TEST_SUCCEED();
}
bool issue1965() {
TEST_START();
std::string str = "{\"query\":\"ah\"}";
@@ -1310,9 +1341,11 @@ namespace object_tests {
}
bool run() {
return issue1979() &&
return testing_to_string() &&
issue1979() &&
issue1977() &&
#if SIMDJSON_EXCEPTIONS
testing_to_string_exception() &&
issue1965() &&
#endif
issue1974a() &&
@@ -23,6 +23,23 @@ bool fatal_error() {
ASSERT_FALSE(doc.is_alive());
TEST_SUCCEED();
}
bool to_string_object() {
TEST_START();
auto json = R"({"\u0062\u0065\u0062\u0065": 2} })"_padded;
ondemand::parser parser;
ondemand::document doc;
auto error = parser.iterate(json).get(doc);
if(error) { return false; }
ondemand::object object;
error = doc.get_object().get(object);
if(error) { return false; }
for (auto field : object) {
std::string key;
error = field.unescaped_key().get(key);
if(error) { return false; }
}
return true;
}
bool simplepad() {
std::string json = "[1]";
+1 -40
View File
@@ -6,6 +6,7 @@
#include "simdjson.h"
#include "cast_tester.h"
#include "test_macros.h"
#include "test_main.h"
template<typename T, typename F>
bool test_ondemand(simdjson::ondemand::parser &parser, const simdjson::padded_string &json, const F& f) {
@@ -43,45 +44,5 @@ bool test_ondemand_doc(const simdjson::padded_string &json, const F& f) {
const size_t AMAZON_CELLPHONES_NDJSON_DOC_COUNT = 793;
#define SIMDJSON_SHOW_DEFINE(x) printf("%s=%s\n", #x, SIMDJSON_STRINGIFY(x))
template<typename F>
int test_main(int argc, char *argv[], const F& test_function) {
std::cout << std::unitbuf;
int c;
while ((c = getopt(argc, argv, "a:")) != -1) {
switch (c) {
case 'a': {
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
if (!impl) {
std::fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
return EXIT_FAILURE;
}
simdjson::get_active_implementation() = impl;
break;
}
default:
std::fprintf(stderr, "Unexpected argument %c\n", c);
return EXIT_FAILURE;
}
}
// this is put here deliberately to check that the documentation is correct (README),
// should this fail to compile, you should update the documentation:
if (simdjson::get_active_implementation()->name() == "unsupported") {
std::printf("unsupported CPU\n");
std::abort();
}
// We want to know what we are testing.
std::cout << "builtin_implementation -- " << simdjson::builtin_implementation()->name() << std::endl;
std::cout << "------------------------------------------------------------" << std::endl;
std::cout << "Running tests." << std::endl;
if (test_function()) {
std::cout << "Success!" << std::endl;
return EXIT_SUCCESS;
} else {
std::cerr << "FAILED." << std::endl;
return EXIT_FAILURE;
}
}
#endif // ONDEMAND_TEST_ONDEMAND_H
+15 -1
View File
@@ -46,6 +46,21 @@ simdjson_inline simdjson::error_code to_error_code(const simdjson::simdjson_resu
return result.error();
}
template <typename T>
std::ostream &operator<<(std::ostream &os, const std::vector<T> &vec) {
os << "["; // Start with opening bracket
if (!vec.empty()) {
// Print first element without leading comma
os << static_cast<uint16_t>(vec[0]);
// Print remaining elements with commas
for (size_t i = 1; i < vec.size(); ++i) {
os << ", " << static_cast<uint16_t>(vec[i]);
}
}
os << "]"; // End with closing bracket
return os;
}
template<typename T>
simdjson_inline bool assert_success(const T &actual, const char *operation = "result") {
simdjson::error_code error = to_error_code(actual);
@@ -124,5 +139,4 @@ simdjson_inline bool assert_iterate_error(T &arr, simdjson::error_code expected,
#define RUN_TEST(ACTUAL) do { if (!(ACTUAL)) { return false; } } while (0);
#define TEST_FAIL(MESSAGE) do { std::cerr << "FAIL: " << (MESSAGE) << std::endl; return false; } while (0);
#define TEST_SUCCEED() do { return true; } while (0);
#endif // TEST_MACROS_H
+50
View File
@@ -0,0 +1,50 @@
#ifndef TEST_MAIN_H
#define TEST_MAIN_H
#include <cstdio>
#include <cstdlib>
#include <iostream>
#include <unistd.h>
#include "simdjson.h"
// technically not a macro, but we want to share this function across tests for both ondemand and dom.
template<typename F>
int test_main(int argc, char *argv[], const F& test_function) {
std::cout << std::unitbuf;
int c;
while ((c = getopt(argc, argv, "a:")) != -1) {
switch (c) {
case 'a': {
const simdjson::implementation *impl = simdjson::get_available_implementations()[optarg];
if (!impl) {
std::fprintf(stderr, "Unsupported architecture value -a %s\n", optarg);
return EXIT_FAILURE;
}
simdjson::get_active_implementation() = impl;
break;
}
default:
std::fprintf(stderr, "Unexpected argument %c\n", c);
return EXIT_FAILURE;
}
}
// this is put here deliberately to check that the documentation is correct (README),
// should this fail to compile, you should update the documentation:
if (simdjson::get_active_implementation()->name() == "unsupported") {
std::printf("unsupported CPU\n");
std::abort();
}
// We want to know what we are testing.
std::cout << "builtin_implementation -- " << simdjson::builtin_implementation()->name() << std::endl;
std::cout << "------------------------------------------------------------" << std::endl;
std::cout << "Running tests." << std::endl;
if (test_function()) {
std::cout << "Success!" << std::endl;
return EXIT_SUCCESS;
} else {
std::cerr << "FAILED." << std::endl;
return EXIT_FAILURE;
}
}
#endif // TEST_MAIN_H