Compare commits

..

8 Commits

Author SHA1 Message Date
John Keiser f56226470c Add ability to handle 8 UTF-8 SIMD blocks at a time 2024-08-18 14:31:12 -07:00
John Keiser 3225226ddc Make UTF-8 validation step by step 2024-08-18 14:31:12 -07:00
John Keiser 2726f4543e Create simd bitmask builder 2024-08-18 14:29:52 -07:00
John Keiser 4c1b0a41d8 Enable SIMDJSON_SINGLEHEADER=OFF in VS Code
With singleheader on, clangd can't find the right
include files.
2024-08-18 14:21:00 -07:00
John Keiser ef563a4b09 Make simdjson compile again 2024-08-18 11:21:26 -07:00
Daniel Lemire 7a9ff93388 [no-ci] Update README.md 2024-08-15 12:52:47 -04:00
Sasha Lopoukhine fc61d7c7ba Fix ndjson spec link (#2234)
* fix ndjson spec link

The link in the readme of parse_many links to a casino spam site

* fix link
2024-08-10 10:11:05 -04:00
Daniel Lemire d506af0a79 fix: add tests related to issue 2227 (#2229)
* fix: add tests related to issue 2227

* avoiding name clash

* pedantic fix

* deprecate rvalue get on document

* selectively deprecating
2024-08-07 20:15:06 -04:00
56 changed files with 5437 additions and 9475 deletions
+9
View File
@@ -25,6 +25,7 @@ CompileFlags:
Diagnostics:
Suppress:
- pp_including_mainfile_in_preamble
- unused-includes
---
# Amalgamated files that require or partly define an implementation
If:
@@ -44,3 +45,11 @@ Diagnostics:
Suppress:
- pragma_attribute_no_pop_eof
- pragma_attribute_stack_mismatch
---
# clang 18
If:
PathMatch:
- dependencies/.cache/json11/json11.cpp
CompileFlags:
Add:
- -Wno-unqualified-std-cast-call
-29
View File
@@ -1,29 +0,0 @@
name: Ubuntu ppc64le (GCC 11)
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: uraimo/run-on-arch-action@v2
name: Test
id: runcmd
with:
arch: aarch64
distro: ubuntu_latest
githubToken: ${{ github.token }}
install: |
apt-get update -q -y
apt-get install -y cmake make g++
run: |
cmake -DSIMDJSON_SANITIZE_UNDEFINED=ON -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
cmake --build build -j=2
ctest --output-on-failure --test-dir build
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
fuzz-seconds: 600
dry-run: false
- name: Upload Crash
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v1
if: failure() && steps.build.outcome == 'success'
with:
name: artifacts
@@ -24,7 +24,7 @@ jobs:
echo "no trailing whitespace found, good!"
fi
- name: Archive whitespace patch
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v2
if: always()
with:
name: whitespace-patch
+1 -1
View File
@@ -22,7 +22,7 @@ jobs:
- msystem: "MINGW64"
install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
type: Debug
- msystem: "MINGW64"
- msystem: "MINGW64"
install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc
type: RelWithDebInfo
env:
+1 -1
View File
@@ -24,6 +24,6 @@ jobs:
apt-get update -q -y
apt-get install -y cmake make g++
run: |
cmake -DCMAKE_BUILD_TYPE=Release -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
cmake -DCMAKE_BUILD_TYPE=Release -B build
cmake --build build -j=2
ctest --output-on-failure --test-dir build
+1 -1
View File
@@ -24,6 +24,6 @@ jobs:
apt-get update -q -y
apt-get install -y cmake make g++
run: |
cmake -DCMAKE_BUILD_TYPE=Release -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
cmake -DCMAKE_BUILD_TYPE=Release -B build
cmake --build build -j=2
ctest --output-on-failure --test-dir build
+1 -1
View File
@@ -24,6 +24,6 @@ jobs:
apt-get update -q -y
apt-get install -y cmake make g++
run: |
cmake -DCMAKE_BUILD_TYPE=Release -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
cmake -DCMAKE_BUILD_TYPE=Release -B build
cmake --build build -j=2
ctest --output-on-failure --test-dir build
-22
View File
@@ -1,22 +0,0 @@
name: Ubuntu 24.04 CI (CXX 20)
on: [push, pull_request]
jobs:
ubuntu-build:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
runs-on: ubuntu-24.04
strategy:
matrix:
cxx: [g++-13, clang++-16]
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Prepare
run: cmake -DSIMDJSON_CXX_STANDARD=20 -DSIMDJSON_DEVELOPER_MODE=ON -B build
env:
CXX: ${{matrix.cxx}}
- name: Build
run: cmake --build build -j=2
- name: Test
run: ctest --output-on-failure --test-dir build
+4 -4
View File
@@ -26,15 +26,15 @@ jobs:
run: |
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -DBUILD_SHARED_LIBS=${{matrix.shared}} -B build
- name: Build Debug
run: cmake --build build --config ${{matrix.build_type}} --verbose
run: cmake --build build --config ${{build_type}} --verbose
- name: Run tests
run: |
cd build
ctest -C ${{matrix.build_type}} -LE explicitonly --output-on-failure
ctest -C ${{build_type}} -LE explicitonly --output-on-failure
- name: Install
run: |
cmake --install build --config ${{matrix.build_type}}
cmake --install build --config ${{build_type}}
- name: Test Installation
run: |
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -B build_install_test tests/installation_tests/find
cmake --build build_install_test --config ${{matrix.build_type}}
cmake --build build_install_test --config ${{build_type}}
-37
View File
@@ -1,37 +0,0 @@
name: VS17-CLANG-CI
on: [push, pull_request]
jobs:
ci:
if: >-
! contains(toJSON(github.event.commits.*.message), '[skip ci]') &&
! contains(toJSON(github.event.commits.*.message), '[skip github]')
name: windows-vs17
runs-on: windows-latest
strategy:
fail-fast: false
matrix:
include:
- {gen: Visual Studio 17 2022, arch: x64, build_type: Debug}
- {gen: Visual Studio 17 2022, arch: x64, build_type: Release}
- {gen: Visual Studio 17 2022, arch: x64, build_type: RelWithDebInfo}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Configure
run: |
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -T ClangCL -DSIMDJSON_DEVELOPER_MODE=ON -DSIMDJSON_COMPETITION=OFF -B build
- name: Build
run: cmake --build build --config ${{matrix.build_type}} --verbose
- name: Run tests
run: |
cd build
ctest -C ${{matrix.build_type}} -LE explicitonly --output-on-failure
- name: Install
run: |
cmake --install build --config ${{matrix.build_type}}
- name: Test Installation
run: |
cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -B build_install_test tests/installation_tests/find
cmake --build build_install_test --config ${{matrix.build_type}}
+6 -18
View File
@@ -109,23 +109,11 @@
"numbers": "cpp",
"semaphore": "cpp",
"stop_token": "cpp",
"cfenv": "cpp",
"format": "cpp",
"xlocmes": "cpp",
"xlocmon": "cpp",
"xlocnum": "cpp",
"xloctime": "cpp",
"xutility": "cpp",
"coroutine": "cpp",
"xfacet": "cpp",
"xhash": "cpp",
"xiosbase": "cpp",
"xlocale": "cpp",
"xlocbuf": "cpp",
"xlocinfo": "cpp",
"xmemory": "cpp",
"xstring": "cpp",
"xtr1common": "cpp",
"xtree": "cpp"
"cfenv": "cpp"
},
"cmake.configureSettings": {
"CMAKE_EXPORT_COMPILE_COMMANDS": "YES",
"SIMDJSON_DEVELOPER_MODE": "ON",
"SIMDJSON_SINGLEHEADER": "OFF"
}
}
+10 -6
View File
@@ -51,6 +51,7 @@ endif()
if(is_top_project)
option(SIMDJSON_DEVELOPER_MODE "Enable targets for developing simdjson" OFF)
option(BUILD_SHARED_LIBS "Build simdjson as a shared library" OFF)
option(SIMDJSON_SINGLEHEADER "Disable singleheader generation" ON)
endif()
include(cmake/handle-deprecations.cmake)
@@ -155,11 +156,13 @@ endif()
include(CMakePackageConfigHelpers)
include(GNUInstallDirs)
install(
FILES singleheader/simdjson.h
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
COMPONENT simdjson_Development
)
if(SIMDJSON_SINGLEHEADER)
install(
FILES singleheader/simdjson.h
DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
COMPONENT simdjson_Development
)
endif()
install(
TARGETS simdjson
@@ -286,8 +289,9 @@ add_subdirectory(tools) ## This needs to be before tests because of cxxopts
# most of the data has been moved to https://github.com/simdjson/simdjson-data
add_subdirectory(jsonexamples)
if(SIMDJSON_SINGLEHEADER)
add_subdirectory(singleheader)
endif()
+2 -2
View File
@@ -201,8 +201,8 @@ For the video inclined, <br />
Funding
-------
The work is supported by the Natural Sciences and Engineering Research Council of Canada under grant
number RGPIN-2017-03910.
The work is supported by the Natural Sciences and Engineering Research Council of Canada under grants
RGPIN-2017-03910 and RGPIN-2024-03787.
[license]: LICENSE
[license img]: https://img.shields.io/badge/License-Apache%202-blue.svg
+1 -1
View File
@@ -13,7 +13,7 @@ struct nlohmann_json {
auto root = nlohmann::json::parse(json.data(), json.data() + json.size());
for (auto tweet : root["statuses"]) {
if (tweet["id"] == find_id) {
result = to_string(tweet["text"]);
result = tweet["text"];
return true;
}
}
+2 -2
View File
@@ -26,8 +26,8 @@ struct nlohmann_json {
}
}
result.text = to_string(top_tweet["text"]);
result.screen_name = to_string(top_tweet["user"]["screen_name"]);
result.text = top_tweet["text"];
result.screen_name = top_tweet["user"]["screen_name"];
return result.retweet_count != -1;
}
};
+7 -210
View File
@@ -17,8 +17,6 @@ An overview of what you need to know to use simdjson, with examples.
- [Using the parsed JSON](#using-the-parsed-json)
- [Using the parsed JSON: additional examples](#using-the-parsed-json-additional-examples)
- [Adding support for custom types](#adding-support-for-custom-types)
- [1. Specialize `simdjson::ondemand::value::get` to get custom types](#1-specialize-simdjsonondemandvalueget-to-get-custom-types)
- [2. Use `tag_invoke` for custom types (C++20)](#2-use-tag_invoke-for-custom-types-c20)
- [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
- [UTF-8 validation (alone)](#utf-8-validation-alone)
- [JSON Pointer](#json-pointer)
@@ -790,25 +788,6 @@ for (ondemand::object points : parser.iterate(points_json)) {
Adding support for custom types
----------------------
There are 2 main ways provided by simdjson to deserialize a value into a custom type:
1. Provide a [**template specialization** for member functions](https://en.cppreference.com/w/cpp/language/template_specialization#Members_of_specializations)
1. Specialize `simdjson::ondemand::document::get` for the whole document
2. Specialize `simdjson::ondemand::value::get` for each value
2. Using `tag_invoke` *(the recommended way if your system supports C++20 or better)*
The differences between the two approaches are as follows:
1. Your compiler needs to support C++20 for `tag_invoke`.
2. The first way limits you to know your type fully, but in the `tag_invoke` way, you can use C++20 concepts and template meta programming as well.
3. Another difference between the two ways is that in `tag_invoke` way you may or may not mark your deserialization function (`tag_invoke` itself) as `noexcept`, but in the
*template specialization* way (the first way) you HAVE TO mark your specialization as `noexcept` which means that you may need to catch exceptions so
that your function does not throw any exception all the while it potentially can (for example any type like `std::string`, `std::vector`, etc. that has an allocator can potentially throw exceptions).
4. Another difference is that the `tag_invoke` way can work on both `document` and `value` but in the first way you have to provide 2 distinct specializations.
5. If you by mistake use both way with each other, the *specialization way* will be used silently due to backward-compatibility.
### 1. Specialize `simdjson::ondemand::value::get` to get custom types
Suppose you have your own types, such as a `Car` struct:
```C++
@@ -842,14 +821,9 @@ type:
```
We may do so by providing additional template definitions to the `ondemand::value` type.
We may start by providing a definition for `std::vector<double>` as follows. Observe
how we guard the code with `#ifndef __cpp_concepts`: that is because the necessary code
is automatically provided by simdjson if C++20 (and concepts) are available.
See [Use `tag_invoke` for custom types](#2-use-tag_invoke-for-custom-types-c20) if you have
C++20 support.
We may start by providing a definition for `std::vector<double>` as follows:
```c++
#ifndef __cpp_concepts // because the code is unnecessary with C++20
template <>
simdjson_inline simdjson_result<std::vector<double>>
simdjson::ondemand::value::get() noexcept {
@@ -861,11 +835,10 @@ simdjson::ondemand::value::get() noexcept {
double val;
error = v.get_double().get(val);
if (error) { return error; }
try { vec.push_back(val); } catch (...) { return simdjson::UNEXPECTED_ERROR; }
vec.push_back(val);
}
return vec;
}
#endif
```
We may then provide support for our `Car` struct:
@@ -883,7 +856,6 @@ simdjson_inline simdjson_result<Car> simdjson::ondemand::value::get() noexcept {
raw_json_string key;
error = field.key().get(key);
if (error) { return error; }
if (key == "make") {
error = field.value().get_string(car.make);
if (error) { return error; }
@@ -895,7 +867,7 @@ simdjson_inline simdjson_result<Car> simdjson::ondemand::value::get() noexcept {
if (error) { return error; }
} else if (key == "tire_pressure") {
error = field.value().get<std::vector<double>>().get(car.tire_pressure);
if (error) { return error; }
if (auto error) { return error; }
}
}
return car;
@@ -921,7 +893,6 @@ struct Car {
std::vector<double> tire_pressure;
};
#ifndef __cpp_concepts // because the code is unnecessary with C++20
template <>
simdjson_inline simdjson_result<std::vector<double>>
simdjson::ondemand::value::get() noexcept {
@@ -937,7 +908,6 @@ simdjson::ondemand::value::get() noexcept {
}
return vec;
}
#endif
template <>
@@ -980,7 +950,7 @@ int main(void) {
ondemand::parser parser;
ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
Car c(val); // an exception may be thrown
Car c(val);
std::cout << c.make << std::endl;
}
direct();
@@ -988,7 +958,7 @@ int main(void) {
}
```
Observe that we require an explicit cast (`Car c(val)` instead of `for (Car c : doc) {`): it is by design. We require explicit casting.
Observe that we require an explicit cast (`Car c(val)` instead of `for (Car c : doc) {`): it is by design.
If you prefer to avoid exceptions, you may modify the `main` function as follows:
@@ -1039,7 +1009,6 @@ struct Car {
std::vector<double> tire_pressure;
};
#ifndef __cpp_concepts // because the code is unnecessary with C++20
template <>
simdjson_inline simdjson_result<std::vector<double>>
simdjson::ondemand::value::get() noexcept {
@@ -1053,7 +1022,7 @@ simdjson::ondemand::value::get() noexcept {
}
return vec;
}
#endif
template <>
simdjson_inline simdjson_result<Car> simdjson::ondemand::document::get() & noexcept {
@@ -1097,178 +1066,6 @@ int main(void) {
}
```
### 2. Use `tag_invoke` for custom types (C++20)
A standard approach to provide support for deserialization is to add a `tag_invoke` function
in your own class:
```C++
/**
* A custom type that we want to parse.
*/
struct Car {
std::string make;
std::string model;
int year = 0;
std::vector<double> tire_pressure;
friend simdjson_result<Car>
tag_invoke(simdjson::deserialize_tag, std::type_identity<Car>, auto &val) {
simdjson::ondemand::object obj;
auto error = val.get_object().get(obj);
if (error) {
return error;
}
Car car{};
// Instead of repeatedly obj["something"], we iterate through the object
// which we expect to be faster.
for (auto field : obj) {
simdjson::ondemand::raw_json_string key;
error = field.key().get(key);
if (error) {
return error;
}
if (key == "make") {
error = field.value().get_string(car.make);
if (error) {
return error;
}
} else if (key == "model") {
error = field.value().get_string(car.model);
if (error) {
return error;
}
} else if (key == "year") {
error = field.value().get(car.year);
if (error) {
return error;
}
} else if (key == "tire_pressure") {
error = field.value().get(car.tire_pressure);
if (error) {
return error;
}
}
}
return car;
}
};
```
Let us explain each argument of `tag_invoke`:
- `simdjson::deserialize_tag`: it is the tag for Customization Point Object (CPO)
- `std::type_identity<T>`: We specify the custom type we want here
- `simdjson::ondemand::value&` or `simdjson::ondemand::document&`: the value or document that we want to deserialize from; you may want to just specify `auto&` to capture both of them.
You can use it like so:
```cpp
simdjson::padded_string json =
R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] })"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
Car c(doc);
std::cout << c.make << std::endl;
```
Observe how we first get an instance of `document` and then we cast.
You can also handle errors explicitly:
```cpp
simdjson::padded_string json =
R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] })"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
Car c;
auto error = doc.get(c);
if(error) { std::cerr << simdjson::error_message(error); return false; }
std::cout << c.make << std::endl;
```
You can also read instances of `Car` from an array or an object:
```cpp
simdjson::padded_string json =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
Car c(val); // an exception may be thrown
std::cout << c.year << std::endl;
}
```
Observe how we first get a generic (`val`) which we cast to `Car`. It is by design: we require
explicit casting. The cast may throw an exception.
Once more, you can handle errors explicitly:
```cpp
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
Car c;
auto error = val.get(c);
if(error) { std::cerr << simdjson::error_message(error) << std::endl; return false; }
}
```
You can also use the custom `Car` type as part of a template such as `std::vector`:
```cpp
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
std::vector<Car> cars(doc);
// visual studio users need an explicit call:
// std::vector<Car> cars = doc.get<std::vector<Car>>();
// because the compiler does not know whether to convert
// doc to an unsigned int or to a vector.
for(Car& c : cars) {
std::cout << c.year << std::endl;
}
```
You can also extend support to arbitrary templates.
Suppose your custom types are `std::unique_ptr<any type>`,
you could add the following `tag_invoke`:
```C++
namespace simdjson {
// This tag_invoke MUST be inside simdjson namespace since we can't make it a friend of unique_ptr
template <typename T>
auto tag_invoke(deserialize_tag, std::type_identity<std::unique_ptr<T>>, auto &val) {
return simdjson_result{std::make_unique<T>(val.template get<T>())};
}
} // namespace simdjson
```
You can also mark your `tag_invoke` function as `noexcept` and we will obey that, unlike the *template specialization* way.
You would use it like this:
```C++
int main() {
auto const json = R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] })"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
std::unique_ptr<Car> c(doc);
std::cout << c->make << std::endl;
return EXIT_SUCCESS;
}
```
Minifying JSON strings without parsing
----------------------
@@ -1546,7 +1343,7 @@ Notice how we can retrieve the exact error condition (in this instance `simdjson
from the exception.
We can write a "quick start" example where we attempt to parse the following JSON file and access some data, without triggering exceptions:
```JSON
```JavaScript
{
"statuses": [
{
+1 -1
View File
@@ -125,7 +125,7 @@ Whitespace Characters:
- **Nothing**
Some official formats **(non-exhaustive list)**:
- [Newline-Delimited JSON (NDJSON)](http://ndjson.org/)
- [Newline-Delimited JSON (NDJSON)](https://github.com/ndjson/ndjson-spec)
- [JSON lines (JSONL)](http://jsonlines.org/)
- [Record separator-delimited JSON (RFC 7464)](https://tools.ietf.org/html/rfc7464) <- Not supported by JsonStream!
- [More on Wikipedia...](https://en.wikipedia.org/wiki/JSON_streaming)
+66 -17
View File
@@ -210,6 +210,7 @@ namespace {
// Bit-specific operations
simdjson_inline simd8<bool> any_bits_set(simd8<uint8_t> bits) const { return vtstq_u8(*this, bits); }
simdjson_inline bool is_ascii() const { return this->max_val() < 0x80u; }
simdjson_inline bool any_bits_set_anywhere() const { return this->max_val() != 0; }
simdjson_inline bool any_bits_set_anywhere(simd8<uint8_t> bits) const { return (*this & bits).any_bits_set_anywhere(); }
template<int N>
@@ -412,6 +413,66 @@ namespace {
}
};
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
static const uint8x16_t BITMASK64_BUILDER_MASK = simdjson_make_uint8x16_t(
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
);
#else
static const uint8x16_t BITMASK64_BUILDER_MASK = {
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
};
#endif
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<4> {
const uint64_t mask;
operator uint64_t() && { return mask; }
}; // struct simd_bitmask64_builder<4>
template<>
struct simd_bitmask64_builder<3> {
const uint8x16_t sum01;
const simd8<bool> val2;
simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
// Add each of the elements next to each other, successively, to stuff each 8 byte mask into one.
uint8x16_t sum23 = vpaddq_u8(val2 & BITMASK64_BUILDER_MASK, val3 & BITMASK64_BUILDER_MASK);
uint8x16_t sum0123 = vpaddq_u8(sum01, sum23);
// This algorithm is actually designed to create a 128-bit mask from 8 16-byte simd masks,
// but since we only want 64 bits, we add the mask to itself (creating the final mask twice).
uint8x16_t sum01230123 = vpaddq_u8(sum0123, sum0123);
return { vgetq_lane_u64(vreinterpretq_u64_u8(sum01230123), 0) };
}
}; // struct simd_bitmask64_builder<3>
template<>
struct simd_bitmask64_builder<2> {
const uint8x16_t sum01;
simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
return { sum01, val2 };
}
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<1> {
const simd8<bool> val0;
simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
return { vpaddq_u8(val0 & BITMASK64_BUILDER_MASK, val1 & BITMASK64_BUILDER_MASK) };
}
}; // struct simd_bitmask64_builder<1>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
return { val };
}
}; // struct simd_bitmask64_builder<0>
template<typename T>
struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -449,23 +510,11 @@ namespace {
}
simdjson_inline uint64_t to_bitmask() const {
#ifdef SIMDJSON_REGULAR_VISUAL_STUDIO
const uint8x16_t bit_mask = simdjson_make_uint8x16_t(
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
);
#else
const uint8x16_t bit_mask = {
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80,
0x01, 0x02, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80
};
#endif
// Add each of the elements next to each other, successively, to stuff each 8 byte mask into one.
uint8x16_t sum0 = vpaddq_u8(this->chunks[0] & bit_mask, this->chunks[1] & bit_mask);
uint8x16_t sum1 = vpaddq_u8(this->chunks[2] & bit_mask, this->chunks[3] & bit_mask);
sum0 = vpaddq_u8(sum0, sum1);
sum0 = vpaddq_u8(sum0, sum0);
return vgetq_lane_u64(vreinterpretq_u64_u8(sum0), 0);
return simd_bitmask64_builder<0>()
.next(this->chunks[0])
.next(this->chunks[1])
.next(this->chunks[2])
.next(this->chunks[3]);
}
simdjson_inline uint64_t eq(const T m) const {
+4
View File
@@ -50,6 +50,8 @@ double from_chars(const char *first, const char* end) noexcept;
#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0)
#if SIMDJSON_REGULAR_VISUAL_STUDIO
// We could use [[deprecated]] but it requires C++14
#define simdjson_deprecated __declspec(deprecated)
#define simdjson_really_inline __forceinline
#define simdjson_never_inline __declspec(noinline)
@@ -88,6 +90,8 @@ double from_chars(const char *first, const char* end) noexcept;
#define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS
#else // SIMDJSON_REGULAR_VISUAL_STUDIO
// We could use [[deprecated]] but it requires C++14
#define simdjson_deprecated __attribute__((deprecated))
#define simdjson_really_inline inline __attribute__((always_inline))
#define simdjson_never_inline inline __attribute__((noinline))
-12
View File
@@ -50,16 +50,4 @@
#endif
#endif
#ifdef __has_include
#if __has_include(<version>)
#include <version>
#endif
#endif
#ifdef __cpp_concepts
#include <utility>
#define SIMDJSON_SUPPORTS_DESERIALIZATION 1
#else // __cpp_concepts
#define SIMDJSON_SUPPORTS_DESERIALIZATION 0
#endif
#endif // SIMDJSON_COMPILER_CHECK_H
@@ -4,7 +4,6 @@
// Stuff other things depend on
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/deserialize.h"
#include "simdjson/generic/ondemand/value_iterator.h"
#include "simdjson/generic/ondemand/value.h"
#include "simdjson/generic/ondemand/logger.h"
@@ -24,13 +23,9 @@
#include "simdjson/generic/ondemand/object_iterator.h"
#include "simdjson/generic/ondemand/serialization.h"
// Deserialization for standard types
#include "simdjson/generic/ondemand/std_deserialize.h"
// Inline definitions
#include "simdjson/generic/ondemand/array-inl.h"
#include "simdjson/generic/ondemand/array_iterator-inl.h"
#include "simdjson/generic/ondemand/value-inl.h"
#include "simdjson/generic/ondemand/document-inl.h"
#include "simdjson/generic/ondemand/document_stream-inl.h"
#include "simdjson/generic/ondemand/field-inl.h"
@@ -43,6 +38,5 @@
#include "simdjson/generic/ondemand/raw_json_string-inl.h"
#include "simdjson/generic/ondemand/serialization-inl.h"
#include "simdjson/generic/ondemand/token_iterator-inl.h"
#include "simdjson/generic/ondemand/value-inl.h"
#include "simdjson/generic/ondemand/value_iterator-inl.h"
@@ -1,114 +0,0 @@
#if SIMDJSON_SUPPORTS_DESERIALIZATION
#ifndef SIMDJSON_ONDEMAND_DESERIALIZE_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_ONDEMAND_DESERIALIZE_H
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/array.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
#include <concepts>
namespace simdjson {
namespace tag_invoke_fn_ns {
void tag_invoke();
struct tag_invoke_fn {
template <typename Tag, typename... Args>
requires requires(Tag tag, Args &&...args) {
tag_invoke(std::forward<Tag>(tag), std::forward<Args>(args)...);
}
constexpr auto operator()(Tag tag, Args &&...args) const
noexcept(noexcept(tag_invoke(std::forward<Tag>(tag),
std::forward<Args>(args)...)))
-> decltype(tag_invoke(std::forward<Tag>(tag),
std::forward<Args>(args)...)) {
return tag_invoke(std::forward<Tag>(tag), std::forward<Args>(args)...);
}
};
} // namespace tag_invoke_fn_ns
inline namespace tag_invoke_ns {
inline constexpr tag_invoke_fn_ns::tag_invoke_fn tag_invoke = {};
} // namespace tag_invoke_ns
template <typename Tag, typename... Args>
concept tag_invocable = requires(Tag tag, Args... args) {
tag_invoke(std::forward<Tag>(tag), std::forward<Args>(args)...);
};
template <typename Tag, typename... Args>
concept nothrow_tag_invocable =
tag_invocable<Tag, Args...> && requires(Tag tag, Args... args) {
{
tag_invoke(std::forward<Tag>(tag), std::forward<Args>(args)...)
} noexcept;
};
template <typename Tag, typename... Args>
using tag_invoke_result =
std::invoke_result<decltype(tag_invoke), Tag, Args...>;
template <typename Tag, typename... Args>
using tag_invoke_result_t =
std::invoke_result_t<decltype(tag_invoke), Tag, Args...>;
template <auto &Tag> using tag_t = std::decay_t<decltype(Tag)>;
struct deserialize_tag;
/// These types are deserializable in a built-in way
template <typename> struct is_builtin_deserializable : std::false_type {};
template <> struct is_builtin_deserializable<int64_t> : std::true_type {};
template <> struct is_builtin_deserializable<uint64_t> : std::true_type {};
template <> struct is_builtin_deserializable<double> : std::true_type {};
template <> struct is_builtin_deserializable<bool> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::array> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::object> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::value> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> : std::true_type {};
template <> struct is_builtin_deserializable<std::string_view> : std::true_type {};
template <typename T>
concept is_builtin_deserializable_v = is_builtin_deserializable<T>::value;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept custom_deserializable = tag_invocable<deserialize_tag, ValT&, T&>;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept deserializable = custom_deserializable<T, ValT> || is_builtin_deserializable_v<T>;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept nothrow_custom_deserializable = nothrow_tag_invocable<deserialize_tag, ValT&, T&>;
// built-in types are noexcept and if an error happens, the value simply gets ignored and the error is returned.
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept nothrow_deserializable = nothrow_custom_deserializable<T, ValT> || is_builtin_deserializable_v<T>;
/// Deserialize Tag
inline constexpr struct deserialize_tag {
using value_type = SIMDJSON_IMPLEMENTATION::ondemand::value;
using document_type = SIMDJSON_IMPLEMENTATION::ondemand::document;
// Customization Point for value
template <typename T>
requires custom_deserializable<T, value_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
return tag_invoke(*this, object, output);
}
// Customization Point for document
template <typename T>
requires custom_deserializable<T, document_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, document_type>) {
return tag_invoke(*this, object, output);
}
} deserialize{};
} // namespace simdjson
#endif // SIMDJSON_ONDEMAND_DESERIALIZE_H
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
@@ -3,19 +3,15 @@
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/array-inl.h"
#include "simdjson/generic/ondemand/array_iterator.h"
#include "simdjson/generic/ondemand/document.h"
#include "simdjson/generic/ondemand/json_iterator-inl.h"
#include "simdjson/generic/ondemand/json_path_to_pointer_conversion.h"
#include "simdjson/generic/ondemand/json_path_to_pointer_conversion-inl.h"
#include "simdjson/generic/ondemand/json_type.h"
#include "simdjson/generic/ondemand/object-inl.h"
#include "simdjson/generic/ondemand/raw_json_string.h"
#include "simdjson/generic/ondemand/value.h"
#include "simdjson/generic/ondemand/value-inl.h"
#include "simdjson/generic/ondemand/array-inl.h"
#include "simdjson/generic/ondemand/json_iterator-inl.h"
#include "simdjson/generic/ondemand/object-inl.h"
#include "simdjson/generic/ondemand/value_iterator-inl.h"
#include "simdjson/generic/ondemand/deserialize.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
@@ -169,20 +165,26 @@ template<> simdjson_inline simdjson_result<int64_t> document::get() & noexcept {
template<> simdjson_inline simdjson_result<bool> document::get() & noexcept { return get_bool(); }
template<> simdjson_inline simdjson_result<value> document::get() & noexcept { return get_value(); }
template<> simdjson_inline error_code document::get(array& out) & noexcept { return get_array().get(out); }
template<> simdjson_inline error_code document::get(object& out) & noexcept { return get_object().get(out); }
template<> simdjson_inline error_code document::get(raw_json_string& out) & noexcept { return get_raw_json_string().get(out); }
template<> simdjson_inline error_code document::get(std::string_view& out) & noexcept { return get_string(false).get(out); }
template<> simdjson_inline error_code document::get(double& out) & noexcept { return get_double().get(out); }
template<> simdjson_inline error_code document::get(uint64_t& out) & noexcept { return get_uint64().get(out); }
template<> simdjson_inline error_code document::get(int64_t& out) & noexcept { return get_int64().get(out); }
template<> simdjson_inline error_code document::get(bool& out) & noexcept { return get_bool().get(out); }
template<> simdjson_inline error_code document::get(value& out) & noexcept { return get_value().get(out); }
template<> simdjson_deprecated simdjson_inline simdjson_result<raw_json_string> document::get() && noexcept { return get_raw_json_string(); }
template<> simdjson_deprecated simdjson_inline simdjson_result<std::string_view> document::get() && noexcept { return get_string(false); }
template<> simdjson_deprecated simdjson_inline simdjson_result<double> document::get() && noexcept { return std::forward<document>(*this).get_double(); }
template<> simdjson_deprecated simdjson_inline simdjson_result<uint64_t> document::get() && noexcept { return std::forward<document>(*this).get_uint64(); }
template<> simdjson_deprecated simdjson_inline simdjson_result<int64_t> document::get() && noexcept { return std::forward<document>(*this).get_int64(); }
template<> simdjson_deprecated simdjson_inline simdjson_result<bool> document::get() && noexcept { return std::forward<document>(*this).get_bool(); }
template<> simdjson_deprecated simdjson_inline simdjson_result<value> document::get() && noexcept { return get_value(); }
template<typename T> simdjson_inline error_code document::get(T &out) & noexcept {
return get<T>().get(out);
}
template<typename T> simdjson_deprecated simdjson_inline error_code document::get(T &out) && noexcept {
return std::forward<document>(*this).get<T>().get(out);
}
#if SIMDJSON_EXCEPTIONS
template <class T>
simdjson_inline document::operator T() noexcept(false) { return get<T>(); }
simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get<T>(); }
template <class T>
simdjson_inline document::operator T() & noexcept(false) { return get<T>(); }
simdjson_inline document::operator array() & noexcept(false) { return get_array(); }
simdjson_inline document::operator object() & noexcept(false) { return get_object(); }
simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); }
@@ -469,7 +471,7 @@ simdjson_inline simdjson_result<T> simdjson_result<SIMDJSON_IMPLEMENTATION::onde
return first.get<T>();
}
template<typename T>
simdjson_inline simdjson_result<T> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get() && noexcept {
simdjson_deprecated simdjson_inline simdjson_result<T> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get() && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::document>(first).get<T>();
}
@@ -485,7 +487,7 @@ simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::do
}
template<> simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get<SIMDJSON_IMPLEMENTATION::ondemand::document>() & noexcept = delete;
template<> simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get<SIMDJSON_IMPLEMENTATION::ondemand::document>() && noexcept {
template<> simdjson_deprecated simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document>::get<SIMDJSON_IMPLEMENTATION::ondemand::document>() && noexcept {
if (error()) { return error(); }
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::document>(first);
}
+24 -64
View File
@@ -4,11 +4,8 @@
#define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/json_iterator.h"
#include "simdjson/generic/ondemand/deserialize.h"
#include "simdjson/generic/ondemand/value.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
@@ -181,39 +178,24 @@ public:
* @returns A value of the given type, parsed from the JSON.
* @returns INCORRECT_TYPE If the JSON value is not the given type.
*/
template <typename T>
simdjson_inline simdjson_result<T> get() &
#if SIMDJSON_SUPPORTS_DESERIALIZATION
noexcept(custom_deserializable<T, document> ? nothrow_custom_deserializable<T, document> : true)
#else
noexcept
#endif
{
static_assert(std::is_default_constructible<T>::value, "Cannot initialize the specified type.");
T out{};
SIMDJSON_TRY(get<T>(out));
return out;
template<typename T> simdjson_inline simdjson_result<T> get() & noexcept {
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
}
/**
* @overload template<typename T> simdjson_result<T> get() & noexcept
*
* We disallow the use tag_invoke CPO on a moved document; it may create UB
* if user uses `ondemand::array` or `ondemand::object` in their custom type.
*
* The member function is still remains specialize-able for compatibility
* reasons, but we completely disallow its use when a tag_invoke customization
* is provided.
*/
template<typename T>
simdjson_inline simdjson_result<T> get() &&
#if SIMDJSON_SUPPORTS_DESERIALIZATION
noexcept(custom_deserializable<T, document> ? nothrow_custom_deserializable<T, document> : true)
#else
noexcept
#endif
{
static_assert(!std::is_same<T, array>::value && !std::is_same<T, object>::value, "You should never hold either an ondemand::array or ondemand::object without a corresponding ondemand::document being alive; that would be Undefined Behaviour.");
return static_cast<document&>(*this).get<T>();
/** @overload template<typename T> simdjson_result<T> get() & noexcept */
template<typename T> simdjson_deprecated simdjson_inline simdjson_result<T> get() && noexcept {
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
}
/**
@@ -227,34 +209,9 @@ public:
* @returns INCORRECT_TYPE If the JSON value is not an object.
* @returns SUCCESS If the parse succeeded and the out parameter was set to the value.
*/
template<typename T>
simdjson_inline error_code get(T &out) &
#if SIMDJSON_SUPPORTS_DESERIALIZATION
noexcept(custom_deserializable<T, document> ? nothrow_custom_deserializable<T, document> : true)
#else
noexcept
#endif
{
#if SIMDJSON_SUPPORTS_DESERIALIZATION
if constexpr (custom_deserializable<T, document>) {
return deserialize(*this, out);
} else {
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
static_cast<void>(out); // to get rid of unused errors
return UNINITIALIZED;
#if SIMDJSON_SUPPORTS_DESERIALIZATION
}
#endif
}
template<typename T> simdjson_inline error_code get(T &out) & noexcept;
/** @overload template<typename T> error_code get(T &out) & noexcept */
template<typename T> simdjson_inline error_code get(T &out) && noexcept;
template<typename T> simdjson_deprecated simdjson_inline error_code get(T &out) && noexcept;
#if SIMDJSON_EXCEPTIONS
/**
@@ -267,7 +224,10 @@ public:
* @returns An instance of type T
*/
template <class T>
explicit simdjson_inline operator T() noexcept(false);
explicit simdjson_inline operator T() & noexcept(false);
template <class T>
explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false);
/**
* Cast this JSON value to an array.
*
@@ -833,7 +793,7 @@ public:
simdjson_inline simdjson_result<bool> is_null() noexcept;
template<typename T> simdjson_inline simdjson_result<T> get() & noexcept;
template<typename T> simdjson_inline simdjson_result<T> get() && noexcept;
template<typename T> simdjson_deprecated simdjson_inline simdjson_result<T> get() && noexcept;
template<typename T> simdjson_inline error_code get(T &out) & noexcept;
template<typename T> simdjson_inline error_code get(T &out) && noexcept;
@@ -316,7 +316,7 @@ private:
friend class document;
friend class json_iterator;
friend struct simdjson_result<ondemand::document_stream>;
friend struct internal::simdjson_result_base<ondemand::document_stream>;
friend struct simdjson::internal::simdjson_result_base<ondemand::document_stream>;
}; // document_stream
} // namespace ondemand
@@ -1,67 +0,0 @@
#pragma once
#ifndef SIMDJSON_ONDEMAND_GENERIC_JSON_PATH_TO_POINTER_CONVERSION_INL_H
#define SIMDJSON_ONDEMAND_GENERIC_JSON_PATH_TO_POINTER_CONVERSION_INL_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_GENERIC_ONDEMAND_JSON_PATH_TO_POINTER_CONVERSION_INL_H
#include "simdjson/generic/ondemand/json_path_to_pointer_conversion.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {}
namespace ondemand {
simdjson_inline std::string json_path_to_pointer_conversion(std::string_view json_path) {
if (json_path.empty() || (json_path.front() != '.' && json_path.front() != '[') {
return "-1"; // Sentinel value to be handled as an error by the caller.
}
std::string result;
// Reserve space to reduce allocations, adjusting for potential increases due
// to escaping.
result.reserve(json_path.size() * 2);
// Skip the initial '.' as it's assumed every path starts with it.
size_t i = 0;
while (i < json_path.length()) {
if (json_path[i] == '.') {
result += '/';
} else if (json_path[i] == '[') {
result += '/';
++i; // Move past the '['
while (i < json_path.length() && json_path[i] != ']') {
if (json_path[i] == '~') {
result += "~0";
} else if (json_path[i] == '/') {
result += "~1";
} else {
result += json_path[i];
}
++i;
}
if (i == json_path.length() || json_path[i] != ']') {
return "-1"; // Returning sentinel value that will be handled as an error by the caller
}
} else {
if (json_path[i] == '~') {
result += "~0";
} else if (json_path[i] == '/') {
result += "~1";
} else {
result += json_path[i];
}
}
++i;
}
return simdjson_result<std::string>(result);
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_ONDEMAND_GENERIC_JSON_PATH_TO_POINTER_CONVERSION_INL_H
@@ -1,22 +0,0 @@
#pragma once
#ifndef SIMDJSON_ONDEMAND_GENERIC_JSON_PATH_TO_POINTER_CONVERSION_H
#define SIMDJSON_ONDEMAND_GENERIC_JSON_PATH_TO_POINTER_CONVERSION_H
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace internal {
/**
* Converts JSONPath to JSON Pointer.
* @param json_path The JSONPath string to be converted.
* @return A string containing the equivalent JSON Pointer.
* @throws simdjson_error If the conversion fails.
*/
simdjson_inline std::string json_path_to_pointer_conversion(std::string_view json_path);
} // namespace internal
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_JSON_PATH_TO_POINTER_CONVERSION_H
+1 -1
View File
@@ -340,7 +340,7 @@ public:
private:
/** @private [for benchmarking access] The implementation to use */
std::unique_ptr<internal::dom_parser_implementation> implementation{};
std::unique_ptr<simdjson::internal::dom_parser_implementation> implementation{};
size_t _capacity{0};
size_t _max_capacity;
size_t _max_depth{DEFAULT_MAX_DEPTH};
@@ -1,196 +0,0 @@
#if SIMDJSON_SUPPORTS_DESERIALIZATION
#ifndef SIMDJSON_ONDEMAND_DESERIALIZE_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_ONDEMAND_DESERIALIZE_H
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/array.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
#include <concepts>
#include <limits>
#include <list>
#include <memory>
#include <string>
#include <vector>
namespace simdjson {
//////////////////////////////
// Number deserialization
//////////////////////////////
template <std::unsigned_integral T>
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept {
using limits = std::numeric_limits<T>;
uint64_t x;
SIMDJSON_TRY(val.get_uint64().get(x));
if (x > (limits::max)()) {
return NUMBER_OUT_OF_RANGE;
}
out = static_cast<T>(x);
return SUCCESS;
}
template <std::floating_point T>
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept {
double x;
SIMDJSON_TRY(val.get_double().get(x));
out = static_cast<T>(x);
return SUCCESS;
}
template <std::signed_integral T>
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept {
using limits = std::numeric_limits<T>;
int64_t x;
SIMDJSON_TRY(val.get_int64().get(x));
if (x > (limits::max)() || x < (limits::min)()) {
return NUMBER_OUT_OF_RANGE;
}
out = static_cast<T>(x);
return SUCCESS;
}
//////////////////////////////
// STL list deserialization
//////////////////////////////
template <typename T, typename AllocT, typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val,
std::list<T, AllocT> &out) noexcept(false) {
// For better error messages, don't use these as constraints on
// the tag_invoke CPO.
static_assert(
deserializable<T, ValT>,
"The specified type inside the list must itself be deserializable");
static_assert(
std::is_default_constructible_v<T>,
"The specified type inside the list must default constructible.");
SIMDJSON_IMPLEMENTATION::ondemand::array arr;
SIMDJSON_TRY(val.get_array().get(arr));
for (auto v : arr) {
if (auto const err = v.get<T>().get(out.emplace_back()); err) {
// If an error occurs, the empty element that we just inserted gets
// removed. We're not using a temp variable because if T is a heavy type,
// we want the valid path to be the fast path and the slow path be the
// path that has errors in it.
static_cast<void>(out.pop_back());
return err;
}
}
return SUCCESS;
}
//////////////////////////////
// std::string deserialization
//////////////////////////////
template <typename CharT,
typename TraitsT,
typename AllocT,
typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val, std::basic_string<CharT, TraitsT, AllocT> &out) noexcept(false) {
using string_type = std::basic_string<CharT, TraitsT, AllocT>;
if constexpr (std::same_as<string_type, string_type>) {
SIMDJSON_TRY(val.get_string(out));
} else {
// todo: optimize performance
std::string tmp;
SIMDJSON_TRY(val.get_string(tmp));
for (auto const ch : tmp) {
out.push_back(ch);
}
}
return SUCCESS;
}
//////////////////////////////
// STL Vector deserialization
//////////////////////////////
/**
* STL containers have several constructors including one that takes a single
* size argument. Thus, some compilers (Visual Studio) will not be able to
* disambiguate between the size and container constructor. Users should
* explicitly specify the type of the container as needed: e.g.,
* doc.get<std::vector<int>>().
*/
template <typename T, typename AllocT, typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val,
std::vector<T, AllocT> &out) noexcept(false) {
// For better error messages, don't use these as constraints on
// the tag_invoke CPO.
static_assert(
deserializable<T, ValT>,
"The specified type inside the vector must itself be deserializable");
static_assert(
std::is_default_constructible_v<T>,
"The specified type inside the vector must default constructible.");
SIMDJSON_IMPLEMENTATION::ondemand::array arr;
SIMDJSON_TRY(val.get_array().get(arr));
for (auto v : arr) {
if (auto const err = v.get<T>().get(out.emplace_back()); err) {
// If an error occurs, the empty element that we just inserted gets
// removed. We're not using a temp variable because if T is a heavy type,
// we want the valid path to be the fast path and the slow path be the
// path that has errors in it.
static_cast<void>(out.pop_back());
return err;
}
}
return SUCCESS;
}
//////////////////////////////
// std::unique_ptr deserialization
//////////////////////////////
/**
* This CPO (Customization Point Object) will help deserialize into
* `unique_ptr`s.
*
* If constructing T is nothrow, this conversion should be nothrow as well since
* we return MEMALLOC if we're not able to allocate memory instead of throwing
* the the error message.
*
* @tparam T The type inside the unique_ptr
* @tparam Deleter The Deleter of the unique_ptr
* @tparam ValT document/value type
* @param val document/value
* @param out output unique_ptr
* @return status of the conversion
*/
template <typename T, typename Deleter, typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val,
std::unique_ptr<T, Deleter>
&out) noexcept(nothrow_deserializable<T, ValT>) {
// For better error messages, don't use these as constraints on
// the tag_invoke CPO.
static_assert(
deserializable<T, ValT>,
"The specified type inside the unique_ptr must itself be deserializable");
static_assert(
std::is_default_constructible_v<T>,
"The specified type inside the unique_ptr must default constructible.");
auto ptr = new (std::nothrow) T();
if (ptr == nullptr) {
return MEMALLOC;
}
SIMDJSON_TRY(val.template get<T>(*ptr));
out.reset(ptr);
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_ONDEMAND_DESERIALIZE_H
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
+9 -20
View File
@@ -6,8 +6,6 @@
#include "simdjson/generic/ondemand/array.h"
#include "simdjson/generic/ondemand/array_iterator.h"
#include "simdjson/generic/ondemand/json_iterator.h"
#include "simdjson/generic/ondemand/json_path_to_pointer_conversion.h"
#include "simdjson/generic/ondemand/json_path_to_pointer_conversion-inl.h"
#include "simdjson/generic/ondemand/json_type.h"
#include "simdjson/generic/ondemand/object.h"
#include "simdjson/generic/ondemand/raw_json_string.h"
@@ -80,7 +78,6 @@ simdjson_inline simdjson_result<bool> value::get_bool() noexcept {
simdjson_inline simdjson_result<bool> value::is_null() noexcept {
return iter.is_null();
}
template<> simdjson_inline simdjson_result<array> value::get() noexcept { return get_array(); }
template<> simdjson_inline simdjson_result<object> value::get() noexcept { return get_object(); }
template<> simdjson_inline simdjson_result<raw_json_string> value::get() noexcept { return get_raw_json_string(); }
@@ -91,16 +88,9 @@ template<> simdjson_inline simdjson_result<uint64_t> value::get() noexcept { ret
template<> simdjson_inline simdjson_result<int64_t> value::get() noexcept { return get_int64(); }
template<> simdjson_inline simdjson_result<bool> value::get() noexcept { return get_bool(); }
template<> simdjson_inline error_code value::get(array& out) noexcept { return get_array().get(out); }
template<> simdjson_inline error_code value::get(object& out) noexcept { return get_object().get(out); }
template<> simdjson_inline error_code value::get(raw_json_string& out) noexcept { return get_raw_json_string().get(out); }
template<> simdjson_inline error_code value::get(std::string_view& out) noexcept { return get_string(false).get(out); }
template<> simdjson_inline error_code value::get(number& out) noexcept { return get_number().get(out); }
template<> simdjson_inline error_code value::get(double& out) noexcept { return get_double().get(out); }
template<> simdjson_inline error_code value::get(uint64_t& out) noexcept { return get_uint64().get(out); }
template<> simdjson_inline error_code value::get(int64_t& out) noexcept { return get_int64().get(out); }
template<> simdjson_inline error_code value::get(bool& out) noexcept { return get_bool().get(out); }
template<typename T> simdjson_inline error_code value::get(T &out) noexcept {
return get<T>().get(out);
}
#if SIMDJSON_EXCEPTIONS
template <class T>
@@ -424,12 +414,6 @@ simdjson_inline simdjson_result<bool> simdjson_result<SIMDJSON_IMPLEMENTATION::o
return first.is_null();
}
template<> simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get<SIMDJSON_IMPLEMENTATION::ondemand::value>(SIMDJSON_IMPLEMENTATION::ondemand::value &out) noexcept {
if (error()) { return error(); }
out = first;
return SUCCESS;
}
template<typename T> simdjson_inline simdjson_result<T> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get() noexcept {
if (error()) { return error(); }
return first.get<T>();
@@ -443,6 +427,11 @@ template<> simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::va
if (error()) { return error(); }
return std::move(first);
}
template<> simdjson_inline error_code simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::get<SIMDJSON_IMPLEMENTATION::ondemand::value>(SIMDJSON_IMPLEMENTATION::ondemand::value &out) noexcept {
if (error()) { return error(); }
out = first;
return SUCCESS;
}
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_type> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::type() noexcept {
if (error()) { return error(); }
@@ -476,7 +465,7 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::number> simdj
template <class T>
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::operator T() noexcept(false) {
if (error()) { throw simdjson_error(error()); }
return first.get<T>();
return static_cast<T>(first);
}
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>::operator SIMDJSON_IMPLEMENTATION::ondemand::array() noexcept(false) {
if (error()) { throw simdjson_error(error()); }
+10 -43
View File
@@ -5,15 +5,12 @@
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/implementation_simdjson_result_base.h"
#include "simdjson/generic/ondemand/value_iterator.h"
#include "simdjson/generic/ondemand/deserialize.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
#include <type_traits>
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
/**
* An ephemeral JSON value returned during iteration. It is only valid for as long as you do
* not access more data in the JSON document.
@@ -38,21 +35,16 @@ public:
* @returns A value of the given type, parsed from the JSON.
* @returns INCORRECT_TYPE If the JSON value is not the given type.
*/
template <typename T>
simdjson_inline simdjson_result<T> get()
#if SIMDJSON_SUPPORTS_DESERIALIZATION
noexcept(custom_deserializable<T, value> ? nothrow_custom_deserializable<T, value> : true)
#else
noexcept
#endif
{
static_assert(std::is_default_constructible<T>::value, "The specified type is not default constructible.");
T out{};
SIMDJSON_TRY(get<T>(out));
return out;
template<typename T> simdjson_inline simdjson_result<T> get() noexcept {
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
}
/**
* Get this value as the given type.
*
@@ -62,32 +54,7 @@ public:
* @returns INCORRECT_TYPE If the JSON value is not an object.
* @returns SUCCESS If the parse succeeded and the out parameter was set to the value.
*/
template <typename T>
simdjson_inline error_code get(T &out)
#if SIMDJSON_SUPPORTS_DESERIALIZATION
noexcept(custom_deserializable<T, value> ? nothrow_custom_deserializable<T, value> : true)
#else
noexcept
#endif
{
#if SIMDJSON_SUPPORTS_DESERIALIZATION
if constexpr (custom_deserializable<T, value>) {
return deserialize(*this, out);
} else {
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
// Unless the simdjson library or the user provides an inline implementation, calling this method should
// immediately fail.
static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. "
"The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, "
"int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), "
" get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."
" You may also add support for custom types, see our documentation.");
static_cast<void>(out); // to get rid of unused errors
return UNINITIALIZED;
#if SIMDJSON_SUPPORTS_DESERIALIZATION
}
#endif
}
template<typename T> simdjson_inline error_code get(T &out) noexcept;
/**
* Cast this JSON value to an array.
@@ -6,9 +6,9 @@
#include "simdjson/generic/atomparsing.h"
#include "simdjson/generic/numberparsing.h"
#include "simdjson/generic/ondemand/json_iterator.h"
#include "simdjson/generic/ondemand/value_iterator.h"
#include "simdjson/generic/ondemand/json_type-inl.h"
#include "simdjson/generic/ondemand/raw_json_string-inl.h"
#include "simdjson/generic/ondemand/value_iterator.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
+27 -3
View File
@@ -295,6 +295,30 @@ namespace simd {
simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7-N)); }
};
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<2> {
const uint64_t bitmask;
operator uint64_t() && { return bitmask; }
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<1> {
const int bitmask;
simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val) {
uint64_t r_lo = uint32_t(bitmask);
uint64_t r_hi = val.to_bitmask();
return { r_lo | (r_hi << 32) };
}
}; // struct simd_bitmask64_builder<1>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) { return { val.to_bitmask() }; }
}; // struct simd_bitmask64_builder<0>
template<typename T>
struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -322,9 +346,9 @@ namespace simd {
}
simdjson_inline uint64_t to_bitmask() const {
uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask());
uint64_t r_hi = this->chunks[1].to_bitmask();
return r_lo | (r_hi << 32);
return simd_bitmask64_builder<0>()
.next(this->chunks[0])
.next(this->chunks[1]);
}
simdjson_inline simd8<T> reduce_or() const {
+14
View File
@@ -314,6 +314,20 @@ namespace simd {
simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7-N)); }
};
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<1> {
const __mmask64 bitmask;
operator __mmask64() && { return bitmask; }
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) { return { _mm512_movepi8_mask(val) }; }
}; // struct simd_bitmask64_builder<0>
template<typename T>
struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
+32 -7
View File
@@ -298,6 +298,35 @@ namespace simd {
simdjson_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(__lasx_xvslli_b(*this, N)); }
};
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<2> {
const unsigned long int mask01;
operator uint64_t() { return mask01; }
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<1> {
const __m256i mask0;
simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
__m256i mask1 = __lasx_xvmskltz_b(val1);
__m256i mask_tmp = __lasx_xvpickve_w(mask0, 4);
__m256i tmp = __lasx_xvpickve_w(mask1, 4);
__m256i mask01 = __lasx_xvinsve0_w(mask0, mask1, 1);
__m256i mask01_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1);
return { __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask01_tmp, mask01), 0) };
}
}; // struct simd_bitmask64_builder<1>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
return { __lasx_xvmskltz_b(val0) };
}
}; // struct simd_bitmask64_builder<0>
template<typename T>
struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -331,13 +360,9 @@ namespace simd {
}
simdjson_inline uint64_t to_bitmask() const {
__m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]);
__m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]);
__m256i mask_tmp = __lasx_xvpickve_w(mask0, 4);
__m256i tmp = __lasx_xvpickve_w(mask1, 4);
mask0 = __lasx_xvinsve0_w(mask0, mask1, 1);
mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1);
return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0);
return simd_bitmask64_builder<0>()
.next(this->chunks[0])
.next(this->chunks[1]);
}
simdjson_inline simd8<T> reduce_or() const {
+49 -7
View File
@@ -255,6 +255,50 @@ namespace simd {
simdjson_inline simd8<uint8_t> shl() const { return simd8<uint8_t>(__lsx_vslli_b(*this, N)); }
};
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<4> {
const unsigned long int result;
operator uint64_t() { return result; }
}; // struct simd_bitmask64_builder<4>
template<>
struct simd_bitmask64_builder<3> {
const __m128i mask01;
const __m128i mask2;
simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
__m128i mask3 = __lsx_vmskltz_b(val3);
__m128i mask23 = __lsx_vilvl_h(mask3, mask2);
return { __lsx_vpickve2gr_du(__lsx_vilvl_w(mask23, mask01), 0) };
}
}; // struct simd_bitmask64_builder<3>
template<>
struct simd_bitmask64_builder<2> {
const __m128i mask01;
simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
return { mask01, __lsx_vmskltz_b(val2) };
}
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<1> {
const __m128i mask0;
simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
__m128i mask1 = __lsx_vmskltz_b(val1);
return { __lsx_vilvl_h(mask1, mask0) };
}
}; // struct simd_bitmask64_builder<1>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val0) {
return { __lsx_vmskltz_b(val0) };
}
}; // struct simd_bitmask64_builder<0>
template<typename T>
struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -303,13 +347,11 @@ namespace simd {
}
simdjson_inline uint64_t to_bitmask() const {
__m128i mask1 = __lsx_vmskltz_b(this->chunks[0]);
__m128i mask2 = __lsx_vmskltz_b(this->chunks[1]);
__m128i mask3 = __lsx_vmskltz_b(this->chunks[2]);
__m128i mask4 = __lsx_vmskltz_b(this->chunks[3]);
mask1 = __lsx_vilvl_h(mask2, mask1);
mask2 = __lsx_vilvl_h(mask4, mask3);
return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0);
return simd_bitmask64_builder<0>()
.next(this->chunks[0])
.next(this->chunks[1])
.next(this->chunks[2])
.next(this->chunks[3]);
}
simdjson_inline simd8<T> reduce_or() const {
+50 -5
View File
@@ -392,6 +392,51 @@ template <> struct simd8<uint8_t> : base8_numeric<uint8_t> {
}
};
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<4> {
const uint64_t bitmask;
simdjson_inline operator uint64_t() && { return bitmask; }
}; // struct simd_bitmask64_builder<4>
template<>
struct simd_bitmask64_builder<3> {
const int bitmask01;
const int bitmask2;
simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val) {
return {
uint64_t(this->bitmask01) |
(uint64_t(this->bitmask2) << 32) |
(uint64_t(val.to_bitmask()) << 48)
};
}
}; // struct simd_bitmask64_builder<3>
template<>
struct simd_bitmask64_builder<2> {
const int bitmask01;
simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val) {
return { bitmask01, val.to_bitmask() };
}
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<1> {
const int bitmask0;
simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val) {
return { bitmask0 | (val.to_bitmask() << 16) };
}
}; // struct simd_bitmask64_builder<1>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val) {
return { val.to_bitmask() };
}
}; // struct simd_bitmask64_builder<0>
template <typename T> struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
static_assert(NUM_CHUNKS == 4,
@@ -434,11 +479,11 @@ template <typename T> struct simd8x64 {
}
simdjson_inline uint64_t to_bitmask() const {
uint64_t r0 = uint32_t(this->chunks[0].to_bitmask());
uint64_t r1 = this->chunks[1].to_bitmask();
uint64_t r2 = this->chunks[2].to_bitmask();
uint64_t r3 = this->chunks[3].to_bitmask();
return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
return simd_bitmask64_builder<0>()
.next(this->chunks[0])
.next(this->chunks[1])
.next(this->chunks[2])
.next(this->chunks[3]);
}
simdjson_inline uint64_t eq(const T m) const {
+50 -5
View File
@@ -260,6 +260,51 @@ namespace simd {
simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7-N)); }
};
template <int N = 0>
struct simd_bitmask64_builder;
template<>
struct simd_bitmask64_builder<4> {
const uint64_t bitmask;
operator uint64_t() && { return bitmask; }
}; // struct simd_bitmask64_builder<4>
template<>
struct simd_bitmask64_builder<3> {
uint32_t bitmask01;
const int bitmask2;
simdjson_inline simd_bitmask64_builder<4> next(simd8<bool> val3) {
return {
uint64_t(this->bitmask01) |
(uint64_t(this->bitmask2) << 32) |
(uint64_t(val3.to_bitmask()) << 48)
};
}
}; // struct simd_bitmask64_builder<3>
template<>
struct simd_bitmask64_builder<2> {
uint32_t bitmask01;
simdjson_inline simd_bitmask64_builder<3> next(simd8<bool> val2) {
return { bitmask01, val2.to_bitmask() };
}
}; // struct simd_bitmask64_builder<2>
template<>
struct simd_bitmask64_builder<1> {
const int bitmask0;
simdjson_inline simd_bitmask64_builder<2> next(simd8<bool> val1) {
return { uint32_t(bitmask0) | (uint32_t(val1.to_bitmask()) << 16) };
}
}; // struct simd_bitmask64_builder<1>
template<>
struct simd_bitmask64_builder<0> {
simdjson_inline simd_bitmask64_builder<1> next(simd8<bool> val0) {
return { val0.to_bitmask() };
}
}; // struct simd_bitmask64_builder<0>
template<typename T>
struct simd8x64 {
static constexpr int NUM_CHUNKS = 64 / sizeof(simd8<T>);
@@ -293,11 +338,11 @@ namespace simd {
}
simdjson_inline uint64_t to_bitmask() const {
uint64_t r0 = uint32_t(this->chunks[0].to_bitmask() );
uint64_t r1 = this->chunks[1].to_bitmask() ;
uint64_t r2 = this->chunks[2].to_bitmask() ;
uint64_t r3 = this->chunks[3].to_bitmask() ;
return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48);
return simd_bitmask64_builder<0>()
.next(this->chunks[0])
.next(this->chunks[1])
.next(this->chunks[2])
.next(this->chunks[3]);
}
simdjson_inline uint64_t eq(const T m) const {
+1 -13
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2024-07-15 08:51:57 -0400. Do not edit! */
/* auto-generated on 2024-08-01 09:31:50 -0400. Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
@@ -77,18 +77,6 @@
#endif
#endif
#ifdef __has_include
#if __has_include(<version>)
#include <version>
#endif
#endif
#ifdef __cpp_concepts
#include <utility>
#define SIMDJSON_SUPPORTS_DESERIALIZATION 1
#else // __cpp_concepts
#define SIMDJSON_SUPPORTS_DESERIALIZATION 0
#endif
#endif // SIMDJSON_COMPILER_CHECK_H
/* end file simdjson/compiler_check.h */
/* including simdjson/portability.h: #include "simdjson/portability.h" */
+4705 -7941
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -1,6 +1,7 @@
// Stuff other things depend on
#include <generic/stage1/base.h>
#include <generic/stage1/buf_block_reader.h>
#include <generic/stage1/simd_reducer.h>
#include <generic/stage1/json_escape_scanner.h>
#include <generic/stage1/json_string_scanner.h>
#include <generic/stage1/utf8_lookup4_algorithm.h>
+2 -3
View File
@@ -239,7 +239,7 @@ simdjson_inline void json_structural_indexer::step<64>(const uint8_t *block, buf
simdjson_inline void json_structural_indexer::next(const simd::simd8x64<uint8_t>& in, const json_block& block, size_t idx) {
uint64_t unescaped = in.lteq(0x1F);
#if SIMDJSON_UTF8VALIDATION
checker.check_next_input(in);
checker.next(in);
#endif
indexer.write(uint32_t(idx-64), prev_structurals); // Output *last* iteration's structurals to the parser
prev_structurals = block.structural_start();
@@ -343,8 +343,7 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa
return EMPTY;
}
}
checker.check_eof();
return checker.errors();
return std::move(checker).eof();
}
} // namespace stage1
+80
View File
@@ -0,0 +1,80 @@
#ifndef SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
#include <generic/stage1/base.h>
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace {
namespace simd {
/** Incrementally performs a reduce OR operation to accumulate an error. */
template <int N>
struct simd_or_reducer;
template<> struct simd_or_reducer<8> {
simd8<uint8_t> error01234567;
operator simd8<uint8_t>() && { return error01234567; }
};
template<> struct simd_or_reducer<7> {
simd8<uint8_t> error0123;
simd8<uint8_t> error45;
simd8<uint8_t> error6;
simd_or_reducer<8> next(simd8<uint8_t> error7) && { return { error0123 | error45 | error6 | error7 }; }
operator simd8<uint8_t>() && { return error0123 | error45 | error6; }
};
template<> struct simd_or_reducer<6> {
simd8<uint8_t> error0123;
simd8<uint8_t> error45;
// simd8<uint8_t> error_;
simd_or_reducer<7> next(simd8<uint8_t> error6) && { return { error0123, error45, error6 }; }
operator simd8<uint8_t>() && { return error0123 | error45; }
};
template<> struct simd_or_reducer<5> {
simd8<uint8_t> error0123;
// simd8<uint8_t> error__;
simd8<uint8_t> error4;
simd_or_reducer<6> next(simd8<uint8_t> error5) && { return { error0123, error4 | error5 }; }
operator simd8<uint8_t>() && { return error0123 | error4; }
};
template<> struct simd_or_reducer<4> {
simd8<uint8_t> error0123;
// simd8<uint8_t> error__;
// simd8<uint8_t> error_;
operator simd8<uint8_t>() && { return error0123; }
};
template<> struct simd_or_reducer<3> {
// simd8<uint8_t> error____;
simd8<uint8_t> error01;
simd8<uint8_t> error2;
simd_or_reducer<4> next(simd8<uint8_t> error3) && { return { error01 | error2 | error3 }; }
operator simd8<uint8_t>() && { return error01 | error2; }
};
template<> struct simd_or_reducer<2> {
// simd8<uint8_t> error____;
simd8<uint8_t> error01;
// simd8<uint8_t> error_;
simd_or_reducer<3> next(simd8<uint8_t> error2) && { return { error01, error2 }; }
operator simd8<uint8_t>() && { return error01; }
};
template<> struct simd_or_reducer<1> {
// simd8<uint8_t> error____;
// simd8<uint8_t> error__;
simd8<uint8_t> error0;
simd_or_reducer<2> next(simd8<uint8_t> error1) && { return { error0 | error1 }; }
operator simd8<uint8_t>() && { return error0; }
};
template<> struct simd_or_reducer<0> {
simd8<uint8_t> prev_error{};
simd_or_reducer<1> next(simd8<uint8_t> error0) && { return { prev_error | error0 }; }
operator simd8<uint8_t>() && { return prev_error; }
};
} // namespace simd
} // unnamed namespace
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_SRC_GENERIC_STAGE1_SIMD_REDUCER_H
+120 -60
View File
@@ -4,6 +4,7 @@
#define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H
#include <generic/stage1/base.h>
#include <generic/dom_parser_implementation.h>
#include <generic/stage1/simd_reducer.h>
#endif // SIMDJSON_CONDITIONAL_INCLUDE
namespace simdjson {
@@ -11,15 +12,84 @@ namespace SIMDJSON_IMPLEMENTATION {
namespace {
namespace utf8_validation {
using namespace simd;
using namespace simd;
simdjson_inline simd8<uint8_t> check_special_cases(const simd8<uint8_t> input, const simd8<uint8_t> prev1) {
// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
// Bit 1 = Too Long (ASCII followed by continuation)
// Bit 2 = Overlong 3-byte
// Bit 4 = Surrogate
// Bit 5 = Overlong 2-byte
// Bit 7 = Two Continuations
// At its height, this will keep 5 registers around.
template <int N>
struct simd_utf8_checker {
/**
* The current error.
*/
simd_or_reducer<N> error;
/**
* Whether the previous input had incomplete UTF-8 characters at the end.
*/
simd8<uint8_t> prev_incomplete;
/**
* Check the next simd input block.
*/
simdjson_inline simd_utf8_checker<N+1> next(simd8<uint8_t> input, simd8<uint8_t> prev_input) &&;
/**
* Used to wrap back around to the beginning, starting with error and prev_incomplete
* from the end of this block
*/
simdjson_inline simd_utf8_checker<0> next_cycle() && {
return { std::move(error), std::move(prev_incomplete) };
}
/**
* Extracts the validation error from the block.
*/
simdjson_inline error_code eof() && {
if ((prev_incomplete | std::move(error)).any_bits_set_anywhere()) {
return error_code::UTF8_ERROR;
} else {
return error_code::SUCCESS;
}
}
private:
simdjson_inline simd8<uint8_t> check_utf8_bytes(simd8<uint8_t> input, simd8<uint8_t> prev_input) const;
simdjson_inline simd8<uint8_t> is_incomplete(simd8<uint8_t> input) const;
simdjson_inline simd8<uint8_t> check_special_cases(simd8<uint8_t> input, simd8<uint8_t> prev1) const;
simdjson_inline simd8<uint8_t> check_multibyte_lengths(simd8<uint8_t> input, simd8<uint8_t> prev_input, simd8<uint8_t> sc) const;
};
template <int N>
simdjson_inline simd_utf8_checker<N+1> simd_utf8_checker<N>::next(simd8<uint8_t> input, simd8<uint8_t> prev_input) && {
if (simdjson_likely(input.is_ascii())) {
return {
std::move(this->error).next(this->prev_incomplete),
simd8<uint8_t>::zero()
};
} else {
return {
std::move(this->error).next(check_utf8_bytes(input, prev_input)),
is_incomplete(input)
};
}
}
template <int N>
simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_utf8_bytes(simd8<uint8_t> input, simd8<uint8_t> prev_input) const {
// Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
// (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
simd8<uint8_t> prev1 = input.prev<1>(prev_input);
simd8<uint8_t> sc = check_special_cases(input, prev1);
return check_multibyte_lengths(input, prev_input, sc);
}
template <int N>
simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_special_cases(simd8<uint8_t> input, simd8<uint8_t> prev1) const {
// Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII)
// Bit 1 = Too Long (ASCII followed by continuation)
// Bit 2 = Overlong 3-byte
// Bit 4 = Surrogate
// Bit 5 = Overlong 2-byte
// Bit 7 = Two Continuations
constexpr const uint8_t TOO_SHORT = 1<<0; // 11______ 0_______
// 11______ 11______
constexpr const uint8_t TOO_LONG = 1<<1; // 0_______ 10______
@@ -103,8 +173,13 @@ using namespace simd;
);
return (byte_1_high & byte_1_low & byte_2_high);
}
simdjson_inline simd8<uint8_t> check_multibyte_lengths(const simd8<uint8_t> input,
const simd8<uint8_t> prev_input, const simd8<uint8_t> sc) {
template <int N>
simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::check_multibyte_lengths(
simd8<uint8_t> input,
simd8<uint8_t> prev_input,
simd8<uint8_t> sc
) const {
simd8<uint8_t> prev2 = input.prev<2>(prev_input);
simd8<uint8_t> prev3 = input.prev<3>(prev_input);
simd8<uint8_t> must23 = must_be_2_3_continuation(prev2, prev3);
@@ -116,7 +191,8 @@ using namespace simd;
// Return nonzero if there are incomplete multibyte characters at the end of the block:
// e.g. if there is a 4-byte character, but it's 3 bytes from the end.
//
simdjson_inline simd8<uint8_t> is_incomplete(const simd8<uint8_t> input) {
template <int N>
simdjson_inline simd8<uint8_t> simd_utf8_checker<N>::is_incomplete(simd8<uint8_t> input) const {
// If the previous input's last 3 bytes match this, they're too short (they ended at EOF):
// ... 1111____ 111_____ 11______
#if SIMDJSON_IMPLEMENTATION_ICELAKE
@@ -144,59 +220,43 @@ using namespace simd;
struct utf8_checker {
// If this is nonzero, there has been a UTF-8 error.
simd8<uint8_t> error;
// The last input we received
simd8<uint8_t> prev_input_block;
// Whether the last input we received was incomplete (used for ASCII fast path)
simd8<uint8_t> prev_incomplete;
simd_utf8_checker<0> checker;
simd8<uint8_t> prev_input;
//
// Check whether the current bytes are valid UTF-8.
//
simdjson_inline void check_utf8_bytes(const simd8<uint8_t> input, const simd8<uint8_t> prev_input) {
// Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes
// (2, 3, 4-byte leads become large positive numbers instead of small negative numbers)
simd8<uint8_t> prev1 = input.prev<1>(prev_input);
simd8<uint8_t> sc = check_special_cases(input, prev1);
this->error |= check_multibyte_lengths(input, prev_input, sc);
}
simdjson_inline void next(const simd8x64<uint8_t>& input) & {
// you might think that a for-loop would work, but under Visual Studio, it is not good enough.
static_assert(
(simd8x64<uint8_t>::NUM_CHUNKS == 1) ||
(simd8x64<uint8_t>::NUM_CHUNKS == 2) ||
(simd8x64<uint8_t>::NUM_CHUNKS == 4),
"We support one, two or four chunks per 64-byte block."
);
// The only problem that can happen at EOF is that a multibyte character is too short
// or a byte value too large in the last bytes: check_special_cases only checks for bytes
// too large in the first of two bytes.
simdjson_inline void check_eof() {
// If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't
// possibly finish them.
this->error |= this->prev_incomplete;
}
simdjson_inline void check_next_input(const simd8x64<uint8_t>& input) {
if(simdjson_likely(is_ascii(input))) {
this->error |= this->prev_incomplete;
} else {
// you might think that a for-loop would work, but under Visual Studio, it is not good enough.
static_assert((simd8x64<uint8_t>::NUM_CHUNKS == 1)
||(simd8x64<uint8_t>::NUM_CHUNKS == 2)
|| (simd8x64<uint8_t>::NUM_CHUNKS == 4),
"We support one, two or four chunks per 64-byte block.");
SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 1) {
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
} else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
} else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
this->check_utf8_bytes(input.chunks[0], this->prev_input_block);
this->check_utf8_bytes(input.chunks[1], input.chunks[0]);
this->check_utf8_bytes(input.chunks[2], input.chunks[1]);
this->check_utf8_bytes(input.chunks[3], input.chunks[2]);
}
this->prev_incomplete = is_incomplete(input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1]);
this->prev_input_block = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1];
SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 1) {
this->checker = std::move(this->checker)
.next(input.chunks[0], this->prev_input)
.next_cycle();
} else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 2) {
this->checker = std::move(this->checker)
.next(input.chunks[0], this->prev_input)
.next(input.chunks[1], input.chunks[0])
.next_cycle();
} else SIMDJSON_IF_CONSTEXPR (simd8x64<uint8_t>::NUM_CHUNKS == 4) {
this->checker = std::move(this->checker)
.next(input.chunks[0], this->prev_input)
.next(input.chunks[1], input.chunks[0])
.next(input.chunks[2], input.chunks[1])
.next(input.chunks[3], input.chunks[2])
.next_cycle();
}
this->prev_input = input.chunks[simd8x64<uint8_t>::NUM_CHUNKS-1];
}
// do not forget to call check_eof!
simdjson_inline error_code errors() {
return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS;
simdjson_inline error_code eof() && {
// The only problem that can happen at EOF is that a multibyte character is too short
// or a byte value too large in the last bytes: check_special_cases only checks for bytes
// too large in the first of two bytes.
return std::move(this->checker).eof() ? error_code::UTF8_ERROR : error_code::SUCCESS;
}
}; // struct utf8_checker
+4 -4
View File
@@ -21,16 +21,16 @@ bool generic_validate_utf8(const uint8_t * input, size_t length) {
buf_block_reader<64> reader(input, length);
while (reader.has_full_block()) {
simd::simd8x64<uint8_t> in(reader.full_block());
c.check_next_input(in);
c.next(in);
reader.advance();
}
// TODO parse the remainder in sub-blocks (16/32 bytes at a time depending on arch)
uint8_t block[64]{};
reader.get_remainder(block);
simd::simd8x64<uint8_t> in(block);
c.check_next_input(in);
c.next(in);
reader.advance();
c.check_eof();
return c.errors() == error_code::SUCCESS;
return std::move(c).eof() == error_code::SUCCESS;
}
bool generic_validate_utf8(const char * input, size_t length) {
+26 -29
View File
@@ -2,35 +2,32 @@
link_libraries(simdjson)
include_directories(..)
add_subdirectory(compilation_failure_tests)
add_cpp_test(ondemand_log_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_log_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_tostring_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_active_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_array_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_array_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_compilation_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_document_stream_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_error_location_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_json_pointer_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_json_path_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_key_string_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_misc_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_number_in_string_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_readme_examples LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_scalar_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_to_string LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_custom_types_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_custom_types_document_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_stl_types_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_log_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_log_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_tostring_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_active_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_array_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_array_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_compilation_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_document_stream_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_error_location_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_json_pointer_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_json_path_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_key_string_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_misc_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_number_in_string_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_readme_examples LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_scalar_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_to_string LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_twitter_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
if(NOT SIMDJSON_SANITIZE)
add_cpp_test(ondemand_cacheline LABELS ondemand acceptance per_implementation)
endif()
@@ -13,7 +13,8 @@ function(add_dual_compile_test TEST_NAME)
target_compile_definitions(${TEST_NAME}_should_not_compile PRIVATE COMPILATION_TEST_USE_FAILING_CODE=1)
endfunction(add_dual_compile_test)
add_dual_compile_test(iterate_object)
add_dual_compile_test(iterate_array)
add_dual_compile_test(iterate_char_star)
add_dual_compile_test(iterate_string_view)
add_dual_compile_test(iterate_temporary_buffer)
@@ -0,0 +1,34 @@
#include <iostream>
#include "simdjson.h"
using namespace simdjson;
int main() {
auto json = "[1]"_padded;
ondemand::parser parser;
auto f = [](ondemand::parser& p, simdjson::padded_string& jsons) -> ondemand::document {
ondemand::document doc;
auto error = p.iterate(jsons).get(doc);
if(error) { std::abort(); }
return doc;
};
ondemand::array arrayv;
#if COMPILATION_TEST_USE_FAILING_CODE
// Not allowed as this would be unsafe, the document must remain alive.
auto error = f(parser).get_array().get(arrayv);
#else
ondemand::document doc = f(parser, json);
auto error = doc.get_array().get(arrayv);
#endif
if(error) {
std::cout << "Failure" << std::endl;
}
int64_t a = 0;
error = arrayv.at(0).get_int64().get(a);
if(error) {
std::cout << "failure" << std::endl;
}
printf("a = %d\n", (int)a);
return EXIT_SUCCESS;
}
@@ -0,0 +1,34 @@
#include <iostream>
#include "simdjson.h"
using namespace simdjson;
int main() {
auto json = "{\"a\":1}"_padded;
ondemand::parser parser;
auto f = [](ondemand::parser& p, simdjson::padded_string& jsons) -> ondemand::document {
ondemand::document doc;
auto error = p.iterate(jsons).get(doc);
if(error) { std::abort(); }
return doc;
};
ondemand::object objv;
#if COMPILATION_TEST_USE_FAILING_CODE
// Not allowed as this would be unsafe, the document must remain alive.
auto error = f(parser).get_object().get(objv);
#else
ondemand::document doc = f(parser, json);
auto error = doc.get_object().get(objv);
#endif
if(error) {
std::cout << "Failure" << std::endl;
}
int64_t a = 0;
error = objv["a"].get_int64().get(a);
if(error) {
std::cout << "failure" << std::endl;
}
printf("a = %d\n", (int)a);
return EXIT_SUCCESS;
}
@@ -59,6 +59,22 @@ void compilation_test_3() {
}
}
}
// Do not run this, it is only meant to compile
void compilation_test_4() {
const padded_string bogus = ""_padded;
ondemand::parser parser;
int64_t x1 = (int64_t)parser.iterate(bogus);
double x2 = (double)parser.iterate(bogus);
std::string_view x = (std::string_view)parser.iterate(bogus);
uint64_t x3 = (uint64_t)parser.iterate(bogus);
bool x4 = (bool)parser.iterate(bogus);
(void) x1;
(void) x2;
(void) x3;
(void) x4;
(void) x;
}
#endif // SIMDJSON_EXCEPTIONS
int main(void) {
@@ -1,192 +0,0 @@
#include "simdjson.h"
#include "test_ondemand.h"
#include <string>
#include <vector>
#if SIMDJSON_SUPPORTS_DESERIALIZATION
namespace simdjson {
// unique_ptr<T>
template <typename T>
error_code tag_invoke(deserialize_tag, auto &val, std::unique_ptr<T>& out) {
out = std::make_unique<T>(val.template get<T>());
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
namespace doc_custom_types_tests {
#if SIMDJSON_EXCEPTIONS && defined(__cpp_concepts)
struct Car {
std::string make{};
std::string model{};
int64_t year{};
std::vector<double> tire_pressure{};
friend error_code tag_invoke(simdjson::deserialize_tag,
auto &val, Car& car) {
simdjson::ondemand::object obj;
if (auto const error = val.get_object().get(obj)) {
return error;
}
// Instead of repeatedly obj["something"], we iterate through the object
// which we expect to be faster.
for (auto field : obj) {
simdjson::ondemand::raw_json_string key;
if (auto const error = field.key().get(key)) {
return error;
}
if (key == "make") {
if (auto const error = field.value().get_string(car.make)) {
return error;
}
} else if (key == "model") {
if (auto const error = field.value().get_string(car.model)) {
return error;
}
} else if (key == "year") {
if (auto const error = field.value().get_int64().get(car.year)) {
return error;
}
} else if (key == "tire_pressure") {
if (auto const error = field.value().get<std::vector<double>>().get(
car.tire_pressure)) {
return error;
}
}
}
return error_code::SUCCESS;
}
};
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>, simdjson::ondemand::value>, "It should be invocable");
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>, simdjson::ondemand::document>, "Tag_invoke should work with document as well.");
static_assert(simdjson::custom_deserializable<std::vector<Car>, simdjson::ondemand::value>, "It should be invocable");
static_assert(simdjson::custom_deserializable<std::vector<Car>, simdjson::ondemand::document>, "Tag_invoke should work with document as well.");
bool custom_test() {
TEST_START();
auto const json = R"( {
"make": "Toyota",
"model": "Camry",
"year": 2018,
"tire_pressure": [ 40.1, 39.9 ]
} )"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
// creating car directly from "doc":
const std::unique_ptr<Car> car(doc);
if (car->make != "Toyota") {
return false;
}
if (car->model != "Camry") {
return false;
}
if (car->year != 2018) {
return false;
}
if (car->tire_pressure.size() != 2) {
return false;
}
TEST_SUCCEED();
}
bool readme_test() {
TEST_START();
auto const json = R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] })"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
std::unique_ptr<Car> c(doc);
std::cout << c->make << std::endl;
TEST_SUCCEED();
}
bool simple_document_test() {
TEST_START();
simdjson::padded_string json =
R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] })"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
Car c(doc);
std::cout << c.make << std::endl;
ASSERT_EQUAL(c.make, "Toyota");
TEST_SUCCEED();
}
bool custom_test_crazier() {
TEST_START();
simdjson::padded_string json =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
#if SIMDJSON_REGULAR_VISUAL_STUDIO
std::vector<Car> cars = doc.get<std::vector<Car>>();
#else
std::vector<Car> cars(doc);
#endif
for(Car& c : cars) {
std::cout << c.year << std::endl;
if (c.make != "Toyota" && c.make != "Kia") {
return false;
}
if (c.model != "Camry" && c.model != "Soul" && c.model != "Tercel") {
return false;
}
if (c.year != 2018 && c.year != 2012 && c.year != 1999) {
return false;
}
if (c.tire_pressure.size() != 2) {
return false;
}
}
TEST_SUCCEED();
}
bool simple_document_test_no_except() {
TEST_START();
simdjson::padded_string json =
R"( { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] })"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
Car c;
auto error = doc.get(c);
if(error) { std::cerr << simdjson::error_message(error); return false; }
std::cout << c.make << std::endl;
ASSERT_EQUAL(c.make, "Toyota");
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS && defined(__cpp_concepts)
readme_test() &&
custom_test() &&
simple_document_test() &&
simple_document_test_no_except() &&
custom_test_crazier() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace doc_custom_types_tests
int main(const int argc, char *argv[]) {
return test_main(argc, argv, doc_custom_types_tests::run);
}
@@ -1,229 +0,0 @@
#include "simdjson.h"
#include "test_ondemand.h"
#include <string>
#include <vector>
#if SIMDJSON_SUPPORTS_DESERIALIZATION
template <typename T>
struct is_unique_ptr : std::false_type {
};
template <typename T>
struct is_unique_ptr<std::unique_ptr<T>> : std::true_type {
};
template <typename T>
concept is_unique_ptr_v = is_unique_ptr<T>::value;
namespace simdjson {
// this is to demonstrate that we can use concept; otherwise a simple
// type_identity<unique_ptr<T>> as the second argument would have done the job.
//
// This tag_invoke MUST be inside simdjson namespace
template <typename T>
requires is_unique_ptr_v<T>
auto tag_invoke(deserialize_tag, auto &val, T& out) {
using type = typename T::element_type;
out = std::make_unique<type>(val.template get<type>());
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
namespace custom_types_tests {
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION
struct Car {
std::string make{};
std::string model{};
int year{};
std::vector<double> tire_pressure{};
friend simdjson::error_code tag_invoke(simdjson::deserialize_tag, auto &val, Car& car) {
simdjson::ondemand::object obj;
auto error = val.get_object().get(obj);
if (error) {
return error;
}
// Instead of repeatedly obj["something"], we iterate through the object
// which we expect to be faster.
for (auto field : obj) {
simdjson::ondemand::raw_json_string key;
error = field.key().get(key);
if (error) {
return error;
}
if (key == "make") {
error = field.value().get_string(car.make);
if (error) {
return error;
}
} else if (key == "model") {
error = field.value().get_string(car.model);
if (error) {
return error;
}
} else if (key == "year") {
error = field.value().get(car.year);
if (error) {
return error;
}
} else if (key == "tire_pressure") {
error = field.value().get(car.tire_pressure);
if (error) {
return error;
}
}
}
return simdjson::SUCCESS;
}
};
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>>, "It should be deserializable");
bool custom_uniqueptr_test() {
TEST_START();
simdjson::padded_string json =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
std::unique_ptr<Car> c(val);
if (c->make != "Toyota" && c->make != "Kia") {
return false;
}
if (c->model != "Camry" && c->model != "Soul" && c->model != "Tercel") {
return false;
}
if (c->year != 2018 && c->year != 2012 && c->year != 1999) {
return false;
}
if (c->tire_pressure.size() != 2) {
return false;
}
}
TEST_SUCCEED();
}
bool custom_test() {
TEST_START();
simdjson::padded_string json =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
Car c(val);
if (c.make != "Toyota" && c.make != "Kia") {
return false;
}
if (c.model != "Camry" && c.model != "Soul" && c.model != "Tercel") {
return false;
}
if (c.year != 2018 && c.year != 2012 && c.year != 1999) {
return false;
}
if (c.tire_pressure.size() != 2) {
return false;
}
}
TEST_SUCCEED();
}
bool custom_test_crazy() {
TEST_START();
simdjson::padded_string json =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
Car c(val);
std::cout << c.year << std::endl;
if (c.make != "Toyota" && c.make != "Kia") {
return false;
}
if (c.model != "Camry" && c.model != "Soul" && c.model != "Tercel") {
return false;
}
if (c.year != 2018 && c.year != 2012 && c.year != 1999) {
return false;
}
if (c.tire_pressure.size() != 2) {
return false;
}
}
TEST_SUCCEED();
}
bool custom_no_except() {
TEST_START();
simdjson::padded_string json =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
for (auto val : doc) {
Car c;
auto error = val.get(c);
if(error) { std::cerr << simdjson::error_message(error) << std::endl; return false; }
}
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION
custom_test() &&
custom_uniqueptr_test() &&
custom_no_except() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace custom_types_tests
int main(int argc, char *argv[]) {
return test_main(argc, argv, custom_types_tests::run);
}
@@ -164,8 +164,7 @@ namespace json_pointer_tests {
ASSERT_TRUE(is_scalar);
ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE);
std::cout << " checking true"<< std::endl;
ASSERT_SUCCESS(parser.iterate(true_json).get(doc));
ASSERT_SUCCESS(doc.is_scalar().get(is_scalar));
ASSERT_SUCCESS(parser.iterate(true_json).is_scalar().get(is_scalar));
ASSERT_TRUE(is_scalar);
ASSERT_ERROR(doc.at_pointer("").get(val), simdjson::SCALAR_DOCUMENT_AS_VALUE);
std::cout << " checking object"<< std::endl;
+3 -4
View File
@@ -49,7 +49,6 @@ struct Car {
std::vector<double> tire_pressure;
};
#ifndef __cpp_concepts
template <>
simdjson_inline simdjson_result<std::vector<double>>
simdjson::ondemand::value::get() noexcept {
@@ -67,7 +66,7 @@ simdjson::ondemand::value::get() noexcept {
}
return vec;
}
#endif
template <>
@@ -194,8 +193,8 @@ int custom_type_with_exceptions() {
])"_padded;
ondemand::parser parser;
ondemand::document doc = parser.iterate(json);
for (auto value : doc) {
Car c(value);
for (auto val : doc) {
Car c(val);
std::cout << c.make << std::endl;
}
return 0;
@@ -1,48 +0,0 @@
#include "simdjson.h"
#include "test_ondemand.h"
#include <string>
#include <vector>
namespace stl_types {
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION
bool basic_general_madness() {
TEST_START();
simdjson::padded_string json =
R"( [ { "codes": [1.2, 3.4, 5.6, 7.8, 9.0] },
{ "codes": [2.2, 4.4, 6.6, 8.8, 10.0] } ])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
std::vector<std::unique_ptr<float>> codes;
for (auto val : doc) {
simdjson::ondemand::object obj;
SIMDJSON_TRY(val.get_object().get(obj));
obj["codes"].get(codes); // append to it
}
if (codes.size() != 10) {
return false;
}
if (*codes[5] != 2.2f || *codes[7] != 6.6f || *codes[9] != 10.0f) {
return false;
}
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION
basic_general_madness() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace stl_types
int main(int argc, char *argv[]) {
return test_main(argc, argv, stl_types::run);
}