introducing a thread-local parser and removing ranges (#2412)

* introducing a thread-local parser

* adding functionality to release the memory

* some more documentation.

* fixing build

* adding benchmarks for 'from'

* generalizing the code somewhat.

* adding tests, fixing the benchmark (now with arrays and streams), and a
minor update to document_stream

* adding missing files (I forgot to check them).

* We cannot use [[nodiscard]] without guarding it, it is C++17

* fixing the cmake

* marking it as experimental

* removing ranges support (it is too experimental)

* putting back documentation.

* guarding SIMDJSON_CONSTEVAL more carefully.

---------

Co-authored-by: Daniel Lemire <dlemire@lemire.me>
This commit is contained in:
Daniel Lemire
2025-08-13 18:08:25 -04:00
committed by GitHub
parent 626eedc3d8
commit faf921bc7e
19 changed files with 1699 additions and 808 deletions
+8 -1
View File
@@ -37,4 +37,11 @@ endif()
if(SIMDJSON_STATIC_REFLECTION)
add_subdirectory(static_reflect)
endif(SIMDJSON_STATIC_REFLECTION)
endif(SIMDJSON_STATIC_REFLECTION)
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-std=c++20" SIMDJSON_COMPILER_SUPPORTS_CXX20)
if(SIMDJSON_EXCEPTIONS AND SIMDJSON_COMPILER_SUPPORTS_CXX20)
add_subdirectory(from)
endif()
+14
View File
@@ -0,0 +1,14 @@
# Executable
add_executable(from_benchmark from_benchmark.cpp)
# Compile for C++20.
target_compile_features(from_benchmark PRIVATE cxx_std_20)
# Check if -march=native is supported
include(CheckCXXCompilerFlag)
check_cxx_compiler_flag("-march=native" SIMDJSON_SUPPORTS_MARCH_NATIVE)
if(SIMDJSON_SUPPORTS_MARCH_NATIVE)
target_compile_options(from_benchmark PRIVATE -march=native)
endif()
target_include_directories(from_benchmark PRIVATE ${CMAKE_CURRENT_LIST_DIR}/..)
+77
View File
@@ -0,0 +1,77 @@
#ifndef BENCHMARK_HELPERS_H
#define BENCHMARK_HELPERS_H
#include "event_counter.h"
#include <atomic>
event_collector collector;
template <class function_type>
std::pair<event_aggregate, size_t>
bench(const function_type &&function, size_t min_repeat = 10,
size_t min_time_ns = 40'000'000, size_t max_repeat = 10000000) {
size_t N = min_repeat;
if (N == 0) {
N = 1;
}
event_aggregate warm_aggregate{};
for (size_t i = 0; i < N; i++) {
std::atomic_thread_fence(std::memory_order_acquire);
collector.start();
function();
std::atomic_thread_fence(std::memory_order_release);
event_count allocate_count = collector.end();
warm_aggregate << allocate_count;
if ((i + 1 == N) && (warm_aggregate.total_elapsed_ns() < min_time_ns) &&
(N < max_repeat)) {
N *= 10;
}
}
event_aggregate aggregate{};
for (size_t i = 0; i < 10; i++) {
std::atomic_thread_fence(std::memory_order_acquire);
collector.start();
for (size_t i = 0; i < N; i++) {
function();
}
std::atomic_thread_fence(std::memory_order_release);
event_count allocate_count = collector.end();
aggregate << allocate_count;
}
return {aggregate, N};
}
double pretty_print(const std::string &name, size_t num_chars,
std::pair<event_aggregate, size_t> result) {
const auto &agg = result.first;
size_t N = result.second;
num_chars *= N;
printf("%-40s : %8.2f ns %8.2f GB/s", name.c_str(),
agg.elapsed_ns() / num_chars, num_chars / agg.elapsed_ns());
if (collector.has_events()) {
printf(" %8.2f GHz %8.2f cycles/char %8.2f ins./char %8.2f i/c",
agg.cycles() / agg.elapsed_ns(), agg.cycles() / num_chars,
agg.instructions() / num_chars, agg.instructions() / agg.cycles());
}
printf("\n");
return num_chars / agg.elapsed_ns();
}
double pretty_print_array(const std::string &name, size_t num_chars, size_t num_elements,
std::pair<event_aggregate, size_t> result) {
const auto &agg = result.first;
size_t N = result.second;
num_chars *= N;
printf("%-40s : %8.2f ns/char %8.2f ns/value %8.2f GB/s", name.c_str(),
agg.elapsed_ns() / num_chars, agg.elapsed_ns() / num_elements, num_chars / agg.elapsed_ns());
if (collector.has_events()) {
printf(" %8.2f GHz %8.2f cycles/char %8.2f ins./char %8.2f ins./value %8.2f i/c",
agg.cycles() / agg.elapsed_ns(), agg.cycles() / num_chars,
agg.instructions() / num_chars, agg.instructions() / num_elements, agg.instructions() / agg.cycles());
}
printf("\n");
return num_chars / agg.elapsed_ns();
}
#endif
+146
View File
@@ -0,0 +1,146 @@
#ifndef CAR_HELPERS_H
#define CAR_HELPERS_H
#include <iomanip>
#include <random>
#include <sstream>
#include <vector>
#include <simdjson.h>
struct Car {
std::string make;
std::string model;
int64_t year; // We deliberately do not include the tire pressure.
};
using namespace simdjson;
std::string random_car_json() {
static const std::vector<std::string> makes = {"Toyota", "Honda", "Ford",
"BMW", "Mazda"};
static const std::vector<std::string> models = {"Camry", "Civic", "Focus",
"320i", "3"};
static thread_local std::mt19937 rng{std::random_device{}()};
std::uniform_int_distribution<int> make_dist(0, makes.size() - 1);
std::uniform_int_distribution<int> model_dist(0, models.size() - 1);
std::uniform_int_distribution<int> year_dist(2000, 2025);
std::uniform_real_distribution<double> pressure_dist(30.0, 45.0);
std::ostringstream oss;
oss << R"({ "make": ")" << makes[make_dist(rng)] << R"(", "model": ")"
<< models[model_dist(rng)] << R"(", "year": )" << year_dist(rng)
<< R"(, "tire_pressure": [ )" << std::fixed << std::setprecision(1)
<< pressure_dist(rng) << ", " << pressure_dist(rng) << R"( ] })";
return oss.str();
}
std::string generate_car_json_stream(size_t N) {
std::ostringstream oss;
for (size_t i = 0; i < N; ++i) {
oss << random_car_json();
oss << "\n";
}
return oss.str();
}
std::string generate_car_json_array(size_t N) {
std::ostringstream oss;
oss << "[\n";
for (size_t i = 0; i < N; ++i) {
oss << random_car_json();
if (i < N - 1) {
oss << ",";
}
oss << "\n";
}
oss << "]";
return oss.str();
}
// We want to maximize C++ portability, but this is not needed with static
// reflection (C++26).
template <>
simdjson_inline simdjson_result<Car>
simdjson::ondemand::document::get() & noexcept {
ondemand::object obj;
auto error = get_object().get(obj);
if (error) {
return error;
}
Car car;
if ((error = obj["make"].get_string(car.make))) {
return error;
}
if ((error = obj["model"].get_string(car.model))) {
return error;
}
if ((error = obj["year"].get_int64().get(car.year))) {
return error;
}
return car;
}
template <>
simdjson_inline simdjson_result<Car> simdjson::ondemand::value::get() noexcept {
ondemand::object obj;
auto error = get_object().get(obj);
if (error) {
return error;
}
Car car;
if ((error = obj["make"].get_string(car.make))) {
return error;
}
if ((error = obj["model"].get_string(car.model))) {
return error;
}
if ((error = obj["year"].get_int64().get(car.year))) {
return error;
}
return car;
}
template <>
simdjson_inline simdjson_result<Car>
simdjson::ondemand::document_reference::get() & noexcept {
ondemand::object obj;
auto error = get_object().get(obj);
if (error) {
return error;
}
Car car;
if ((error = obj["make"].get_string(car.make))) {
return error;
}
if ((error = obj["model"].get_string(car.model))) {
return error;
}
if ((error = obj["year"].get_int64().get(car.year))) {
return error;
}
return car;
}
template <>
simdjson_inline simdjson_result<Car>
simdjson::ondemand::document_reference::get() && noexcept {
ondemand::object obj;
auto error = get_object().get(obj);
if (error) {
return error;
}
Car car;
if ((error = obj["make"].get_string(car.make))) {
return error;
}
if ((error = obj["model"].get_string(car.model))) {
return error;
}
if ((error = obj["year"].get_int64().get(car.year))) {
return error;
}
return car;
}
#endif
+132
View File
@@ -0,0 +1,132 @@
// We are going to assume C++20.
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <simdjson.h>
#include <string>
#include "benchmark_helpers.h"
#include "car_helpers.h"
using namespace simdjson;
void run_benchmarks() {
printf("Running benchmarks on tiny input...\n");
simdjson::padded_string json =
R"({ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9 ] })"_padded;
volatile size_t dummy = 0;
ondemand::parser parser;
// Classic On-Demand
pretty_print("simdjson classic", json.size(),
bench([&json, &dummy, &parser]() {
ondemand::document doc = parser.iterate(json);
Car car = doc.get<Car>();
dummy = dummy + car.year;
}));
// Benchmark simdjson::from() without parser
pretty_print("simdjson::from<Car>() (no parser)", json.size(),
bench([&json, &dummy]() {
Car car = simdjson::from(json);
dummy = dummy + car.year;
}));
// Benchmark simdjson::from() with parser
pretty_print("simdjson::from<Car>() (with parser)", json.size(),
bench([&json, &dummy, &parser]() {
Car car = simdjson::from(parser, json);
dummy = dummy + car.year;
}));
}
void run_array_benchmarks() {
printf("Running benchmarks on large array...\n");
size_t N = 1'000'000;
simdjson::padded_string json = generate_car_json_array(N);
volatile size_t dummy = 0;
ondemand::parser parser;
// Classic On-Demand
pretty_print_array("simdjson classic", json.size(), N,
bench([&json, &dummy, &parser]() {
dummy = 0;
ondemand::document doc = parser.iterate(json);
ondemand::array array = doc.get_array();
for (auto value : array) {
Car car = value.get<Car>();
dummy = dummy + car.year;
}
}));
// from with array
pretty_print_array("simdjson::from(json).array()", json.size(), N,
bench([&json, &dummy, &parser]() {
dummy = 0;
for (auto value : simdjson::from(json).array()) {
Car car = value.get<Car>();
dummy = dummy + car.year;
}
}));
#if SIMDJSON_SUPPORTS_RANGES
// Benchmark simdjson::from() without parser (EXPERIMENTAL)
pretty_print_array("simdjson::from<Car>() (experimental, no parser)",
json.size(), N, bench([&json, &dummy]() {
dummy = 0;
for (Car car :
simdjson::from(json) | simdjson::as<Car>()) {
dummy = dummy + car.year;
}
}));
// Benchmark simdjson::from() with parser (EXPERIMENTAL)
pretty_print_array("simdjson::from<Car>() (experimental, with parser)",
json.size(), N, bench([&json, &dummy, &parser]() {
dummy = 0;
for (Car car : simdjson::from(parser, json) |
simdjson::as<Car>()) {
dummy = dummy + car.year;
}
}));
#endif // SIMDJSON_SUPPORTS_RANGES
}
void run_stream_benchmarks() {
printf("Running stream benchmarks...\n");
simdjson::padded_string json = generate_car_json_stream(1'000'000);
volatile size_t dummy = 0;
ondemand::parser parser;
// Classic On-Demand
pretty_print("simdjson classic", json.size(),
bench([&json, &dummy, &parser]() {
ondemand::document_stream stream = parser.iterate_many(json);
for (auto doc : stream) {
Car car = doc.get<Car>();
dummy = dummy + car.year;
}
}));
// from_many
pretty_print("simdjson with thread local", json.size(),
bench([&json, &dummy, &parser]() {
ondemand::document_stream stream =
ondemand::parser::get_parser().iterate_many(json);
for (auto doc : stream) {
Car car = doc.get<Car>();
dummy = dummy + car.year;
}
}));
}
int main() {
for (size_t trial = 0; trial < 3; trial++) {
printf("Trial %zu:\n", trial + 1);
run_array_benchmarks();
run_stream_benchmarks();
run_benchmarks();
printf("\n");
}
return EXIT_SUCCESS;
}
+23 -59
View File
@@ -180,6 +180,18 @@ auto json = padded_string::load("twitter.json"); // load JSON file 'twitter.json
ondemand::document doc = parser.iterate(json); // position a pointer at the beginning of the JSON data
```
If you prefer not to create your own `ondemand::parser` instance, you can access
a thread-local version by calling `ondemand::parser.get_parser()`.
```c++
ondemand::document doc = ondemand::parser.get_parser().iterate(json);
```
However, you should be careful because a parser instance can only be used for one
document at a time, thus it is only applicable when you are only parsing one
document per thread at any one time.
You can also create a padded string---and call `iterate()`:
```c++
@@ -1351,57 +1363,25 @@ that are not made by Toyota.
For even more convenience, you can do it directly without a document instance like so:
```cpp
simdjson::ondemand::parser parser;
Car car = simdjson::from(parser, json_car);
Car car = simdjson::from(json);
```
We strongly encourage you to rely on an parser instance (`simdjson::ondemand::parser`) which you reuse. However, if performance is not a concern, you can omit the declaration
of a parser instance like so:
You can also use the `simdjson::from` syntax to iterate over an array.
```cpp
Car car = simdjson::from(json);
for(auto val : simdjson::from(json).array()) {
Car c = val.get<Car>(); // ...
}
```
Standard STL types are supported:
```cpp
simdjson::ondemand::parser parser;
std::map<std::string, std::string> obj =
simdjson::from(parser, R"({"key": "value"})"_padded);
simdjson::from(R"({"key": "value"})"_padded);
```
You can also use C++20 ranges to iterate over an array:
```cpp
simdjson::padded_string json_cars =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
for (Car car : simdjson::from(parser, json_cars) | simdjson::as<Car>()) {
if (car.year < 1998) {
return false;
}
}
```
Again, if performance is not a concern, you can omit the parser instance.
```cpp
for (Car car : simdjson::from(json_cars) | simdjson::as<Car>()) {
if (car.year < 1998) {
return false;
}
}
```
### 3. Using static reflection (C++26)
If you have a C++26 compatible compiler, you can compile
@@ -1426,33 +1406,17 @@ Car c = doc.get<Car>();
Just like when using `tag_invoke` for custom types (but without the `tag_invoke` code), you can parse a class instance directly without a parser instance:
```cpp
simdjson::ondemand::parser parser;
Car car = simdjson::from(parser, json);
Car car = simdjson::from(json);
```
Similarly, you can also use C++20 ranges to iterate over an array:
You can also use the `simdjson::from` syntax to iterate over an array.
```cpp
simdjson::padded_string json_cars =
R"( [ { "make": "Toyota", "model": "Camry", "year": 2018,
"tire_pressure": [ 40.1, 39.9 ] },
{ "make": "Kia", "model": "Soul", "year": 2012,
"tire_pressure": [ 30.1, 31.0 ] },
{ "make": "Toyota", "model": "Tercel", "year": 1999,
"tire_pressure": [ 29.8, 30.0 ] }
])"_padded;
simdjson::ondemand::parser parser;
for (Car car : simdjson::from(, parserjson_cars) | simdjson::as<Car>()) {
if (car.year < 1998) {
return false;
}
}
for(auto val : simdjson::from(json).array()) {
Car c = val.get<Car>(); // ...
}
```
When using the `simdjson::from` syntax, you can also omit the parser instance,
for more convenience. However, we strongly encourage you to create a single parser
instance that is reused, as it leads to better performance.
You can also automatically serialize the `Car` instance to a JSON string, see
our [Builder documentation](builder.md).
+2 -2
View File
@@ -94,11 +94,11 @@
#endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED)
#if !defined(SIMDJSON_CONSTEVAL)
#if defined(__cpp_consteval) && __cpp_consteval >= 201811L
#if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L
#define SIMDJSON_CONSTEVAL 1
#else
#define SIMDJSON_CONSTEVAL 0
#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L
#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L
#endif // !defined(SIMDJSON_CONSTEVAL)
#endif // SIMDJSON_COMPILER_CHECK_H
+30 -159
View File
@@ -4,76 +4,11 @@
#include "simdjson/ondemand.h"
#include <optional>
#ifdef __cpp_lib_ranges
#include <ranges>
#endif
namespace simdjson {
struct [[nodiscard]] auto_iterator_end {};
/**
* A Wrapper for simdjson_result<ondemand::array_iterator> in order to make it
* compatible with ranges (to satisfy std::ranges::input_range).
*/
struct [[nodiscard]] auto_iterator {
using iterator_category = std::forward_iterator_tag;
using type = simdjson_result<ondemand::array_iterator>;
using value_type = simdjson_result<ondemand::value>; // type::value_type
using reference = value_type &;
using const_reference = const value_type &;
using difference_type = std::ptrdiff_t;
struct auto_iterator_storage {
type m_iter{};
mutable value_type m_value{};
};
private:
auto_iterator_storage *m_storage = nullptr;
public:
constexpr auto_iterator() noexcept = default;
explicit auto_iterator(auto_iterator_storage &storage) noexcept
: m_storage{&storage} {};
auto_iterator(auto_iterator const &) = default;
auto_iterator(auto_iterator &&) = default;
auto_iterator &operator=(auto_iterator const &) = default;
auto_iterator &operator=(auto_iterator &&) noexcept = default;
~auto_iterator() = default;
reference operator*() const noexcept { return m_storage->m_value; }
reference operator*() noexcept { return m_storage->m_value; }
auto_iterator &operator++() noexcept {
++m_storage->m_iter;
m_storage->m_value =
m_storage->m_iter.at_end() || m_storage->m_iter.error() != SUCCESS
? value_type{}
: *m_storage->m_iter;
return *this;
}
auto_iterator operator++(int) noexcept {
auto_iterator const tmp = *this;
operator++();
return tmp;
}
[[nodiscard]] bool operator==(auto_iterator const &other) const noexcept {
return m_storage == other.m_storage &&
m_storage->m_iter == other.m_storage->m_iter;
}
[[nodiscard]] bool operator==(auto_iterator_end) const noexcept {
return m_storage != nullptr && m_storage->m_iter.at_end();
}
};
template <typename ParserType = ondemand::parser>
struct [[nodiscard]] auto_parser
#if __cpp_lib_ranges
: std::ranges::view_interface<auto_parser<ParserType>>
#endif
template <typename ParserType = ondemand::parser*>
struct auto_parser
{
using value_type = simdjson_result<ondemand::value>;
using size_type = size_t;
@@ -82,17 +17,12 @@ struct [[nodiscard]] auto_parser
using const_pointer = const value_type *;
using reference = value_type &;
using const_reference = const value_type &;
using iterator = auto_iterator;
using const_iterator = auto_iterator; // auto_iterator is already const
private:
ParserType m_parser;
ondemand::document m_doc;
error_code m_error{SUCCESS};
// Caching the iterator here:
iterator::auto_iterator_storage iter_storage{};
template <typename T>
static constexpr bool is_nothrow_gettable = requires(ondemand::document doc) {
{ doc.get<T>() } noexcept;
@@ -111,10 +41,6 @@ public:
m_error = m_parser.iterate(str).get(m_doc);
}
explicit auto_parser(padded_string_view const str) noexcept
requires(!std::is_pointer_v<ParserType>)
: auto_parser{ParserType{}, str} {}
// pointer constructors:
explicit auto_parser(std::remove_pointer_t<ParserType> &parser,
ondemand::document &&doc) noexcept
@@ -128,6 +54,10 @@ public:
m_error = m_parser->iterate(str).get(m_doc);
}
explicit auto_parser(padded_string_view const str) noexcept
requires(std::is_pointer_v<ParserType>)
: auto_parser{ondemand::parser::get_parser(), str} {}
explicit auto_parser(ParserType parser, ondemand::document &&doc) noexcept
requires(std::is_pointer_v<ParserType>)
: auto_parser{*parser, std::move(doc)} {}
@@ -139,7 +69,7 @@ public:
~auto_parser() = default;
/// Get the parser
[[nodiscard]] std::remove_pointer_t<ParserType> &parser() noexcept {
simdjson_warn_unused std::remove_pointer_t<ParserType> &parser() noexcept {
if constexpr (std::is_pointer_v<ParserType>) {
return *m_parser;
} else {
@@ -148,7 +78,7 @@ public:
}
template <typename T>
[[nodiscard]] simdjson_inline simdjson_result<T>
simdjson_warn_unused simdjson_inline simdjson_result<T>
result() noexcept(is_nothrow_gettable<T>) {
if (m_error != SUCCESS) {
return m_error;
@@ -157,29 +87,29 @@ public:
return m_doc.get<T>();
}
[[nodiscard]] simdjson_inline simdjson_result<ondemand::array>
simdjson_warn_unused simdjson_inline simdjson_result<ondemand::array>
array() noexcept {
return result<ondemand::array>();
}
[[nodiscard]] simdjson_inline simdjson_result<ondemand::object>
simdjson_warn_unused simdjson_inline simdjson_result<ondemand::object>
object() noexcept {
return result<ondemand::object>();
}
[[nodiscard]] simdjson_inline simdjson_result<ondemand::number>
simdjson_warn_unused simdjson_inline simdjson_result<ondemand::number>
number() noexcept {
return result<ondemand::number>();
}
template <typename T>
[[nodiscard]] simdjson_inline explicit(false)
simdjson_warn_unused simdjson_inline explicit(false)
operator simdjson_result<T>() noexcept(is_nothrow_gettable<T>) {
return result<T>();
}
template <typename T>
[[nodiscard]] simdjson_inline explicit(false) operator T() noexcept(false) {
simdjson_warn_unused simdjson_inline explicit(false) operator T() noexcept(false) {
if (m_error != SUCCESS) {
throw simdjson_error(m_error);
}
@@ -192,7 +122,7 @@ public:
// We also cannot have "operator T&" without manual memory management either.
template <typename T>
[[nodiscard]] simdjson_inline std::optional<T>
simdjson_warn_unused simdjson_inline std::optional<T>
optional() noexcept(is_nothrow_gettable<T>) {
if (m_error != SUCCESS) {
return std::nullopt;
@@ -204,76 +134,25 @@ public:
}
return {std::move(value)};
}
simdjson_inline auto_iterator begin() noexcept {
if (m_error != SUCCESS) {
// Create an iterator with the error
iter_storage.m_iter = iterator::type(m_error);
iter_storage.m_value = value_type{};
return auto_iterator{iter_storage};
}
if (iter_storage.m_iter.error() != SUCCESS &&
!iter_storage.m_iter.at_end()) {
// Try to get the document as an array
ondemand::array arr;
if(auto error = m_doc.get_array().get(arr); error == SUCCESS) {
iter_storage = {.m_iter = iterator::type{arr.begin()},
.m_value = iterator::value_type{
iter_storage.m_iter.at_end() ||
iter_storage.m_iter.error() != SUCCESS
? value_type{}
: *iter_storage.m_iter}};
} else {
// If it's not an array, create an error iterator
iter_storage.m_iter = iterator::type(error);
iter_storage.m_value = value_type{};
}
}
return auto_iterator{iter_storage};
}
simdjson_inline auto_iterator_end end() noexcept { return {}; }
};
#ifdef __cpp_lib_ranges
// For C++20, we implement our own pipe operator since range_adaptor_closure is C++23
static constexpr struct [[nodiscard]] no_errors_adaptor {
[[nodiscard]] bool
operator()(simdjson_result<ondemand::value> const &val) const noexcept {
return val.error() == SUCCESS;
}
template <std::ranges::range Range>
auto operator()(Range &&rng) const noexcept {
return std::forward<Range>(rng) | std::views::filter(*this);
}
} no_errors;
template <typename T = void>
struct [[nodiscard]] to_adaptor {
struct to_adaptor {
/// Convert to T
[[nodiscard]] T
operator()(simdjson_result<ondemand::value> &val) const noexcept {
T operator()(simdjson_result<ondemand::value> &val) const noexcept {
return val.get<T>();
}
/// Make it an adaptor
template <std::ranges::range Range>
auto operator()(Range &&rng) const noexcept {
return std::forward<Range>(rng) | no_errors | std::views::transform(*this);
}
/**
* Parse input string into any object if possible. This function call
* will return an auto_parser that can be used to extract the desired
* object from the input string. The ondemand::parser instance is created
* internally.
*
* *WARNING*: This function call will create a new parser instance each time
* it is called. This has performance implications. We strongly recommend
* that you create a parser instance once and reuse it many times.
* This function uses the simdjson::ondemand::parser::get_parser() instance.
* A parser should only be used for one document at a time.
*/
auto operator()(padded_string_view const str) const noexcept {
return auto_parser{str};
@@ -300,34 +179,26 @@ template <typename T> static constexpr to_adaptor<T> to{};
* Example usage:
*
* ```cpp
* std::map<std::string, std::string> obj =
* simdjson::from(R"({"key": "value"})"_padded);
* ```
*
* This will parse the JSON string and return an object representation. By default, we
* use the simdjson::ondemand::parser::get_parser() instance. A parser instance should
* be used for just one document at a time.
*
* You can also pass you own parser instance:
* ```cpp
* simdjson::ondemand::parser parser;
* std::map<std::string, std::string> obj =
* simdjson::from(parser, R"({"key": "value"})"_padded);
* ```
* The parser instance can be reused.
*
* This will parse the JSON string and return an object representation. The `ondemand::parser`
* instance can be reused. It is also possible to omit the parser instance, in which case a parser
* instance will be created internally, although this can have negative performance consequences.
*/
static constexpr to_adaptor<> from{};
template <typename T = void>
using as = to_adaptor<T>;
// For C++20 ranges without range_adaptor_closure, we need to define pipe operators
template <std::ranges::range Range>
inline auto operator|(Range&& range, const no_errors_adaptor& adaptor) {
return adaptor(std::forward<Range>(range));
}
template <std::ranges::range Range, typename T>
inline auto operator|(Range&& range, const to_adaptor<T>& adaptor) {
return adaptor(std::forward<Range>(range));
}
#endif // __cpp_lib_ranges
} // namespace simdjson
#endif // __cpp_concepts
#endif // SIMDJSON_CONVERT_H
#endif // SIMDJSON_CONVERT_H
@@ -62,7 +62,7 @@ public:
/**
* Check if the array is at the end.
*/
[[nodiscard]] simdjson_inline bool at_end() const noexcept;
simdjson_warn_unused simdjson_inline bool at_end() const noexcept;
private:
value_iterator iter{};
@@ -95,7 +95,7 @@ struct simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> : publ
simdjson_inline bool operator!=(const simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &) const noexcept;
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::array_iterator> &operator++() noexcept;
[[nodiscard]] simdjson_inline bool at_end() const noexcept;
simdjson_warn_unused simdjson_inline bool at_end() const noexcept;
};
} // namespace simdjson
@@ -97,35 +97,35 @@ inline constexpr struct deserialize_tag {
// Customization Point for array
template <typename T>
requires custom_deserializable<T, value_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
return tag_invoke(*this, object, output);
}
// Customization Point for object
template <typename T>
requires custom_deserializable<T, value_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
return tag_invoke(*this, object, output);
}
// Customization Point for value
template <typename T>
requires custom_deserializable<T, value_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
return tag_invoke(*this, object, output);
}
// Customization Point for document
template <typename T>
requires custom_deserializable<T, document_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, document_type>) {
simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, document_type>) {
return tag_invoke(*this, object, output);
}
// Customization Point for document reference
template <typename T>
requires custom_deserializable<T, document_reference_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, document_reference_type>) {
simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, document_reference_type>) {
return tag_invoke(*this, object, output);
}
@@ -182,10 +182,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++
return *this;
}
simdjson_inline bool document_stream::iterator::at_end() const noexcept {
return finished;
}
simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept {
return finished != other.finished;
}
simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept {
return finished == other.finished;
}
simdjson_inline document_stream::iterator document_stream::begin() noexcept {
start();
// If there are no documents, we're finished.
@@ -131,6 +131,7 @@ public:
* Default constructor.
*/
simdjson_inline iterator() noexcept;
simdjson_inline iterator(const iterator &other) noexcept = default;
/**
* Get the current document (or error).
*/
@@ -144,6 +145,7 @@ public:
* @param other the end iterator to compare to.
*/
simdjson_inline bool operator!=(const iterator &other) const noexcept;
simdjson_inline bool operator==(const iterator &other) const noexcept;
/**
* @private
*
@@ -187,6 +189,11 @@ public:
*/
inline error_code error() const noexcept;
/**
* Returns whether the iterator is at the end.
*/
inline bool at_end() const noexcept;
private:
simdjson_inline iterator(document_stream *s, bool finished) noexcept;
/** The document_stream we're iterating through. */
@@ -198,6 +205,7 @@ public:
friend class document_stream;
friend class json_iterator;
};
using iterator = document_stream::iterator;
/**
* Start iterating the documents in the stream.
@@ -145,6 +145,13 @@ inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf,
if(allow_comma_separated && batch_size < len) { batch_size = len; }
return document_stream(*this, buf, len, batch_size, allow_comma_separated);
}
inline simdjson_result<document_stream> parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept {
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); }
return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated);
}
inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size, allow_comma_separated);
}
@@ -189,6 +196,34 @@ simdjson_inline simdjson_warn_unused simdjson_result<std::string_view> parser::u
return result;
}
simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() {
return *parser::get_parser_instance();
}
simdjson_inline bool release_parser() {
auto &parser_instance = parser::get_threadlocal_parser_if_exists();
if (parser_instance) {
parser_instance.reset();
return true;
}
return false;
}
simdjson_inline simdjson_warn_unused std::unique_ptr<ondemand::parser>& parser::get_parser_instance() {
std::unique_ptr<ondemand::parser>& parser_instance = get_threadlocal_parser_if_exists();
if (!parser_instance) {
parser_instance.reset(new ondemand::parser());
}
return parser_instance;
}
simdjson_inline simdjson_warn_unused std::unique_ptr<ondemand::parser>& parser::get_threadlocal_parser_if_exists() {
// @the-moisrex points out that this could be implemented with std::optional (C++17).
thread_local std::unique_ptr<ondemand::parser> parser_instance = nullptr;
return parser_instance;
}
} // namespace ondemand
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
+34 -3
View File
@@ -7,6 +7,7 @@
#endif // SIMDJSON_CONDITIONAL_INCLUDE
#include <memory>
#include <thread>
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
@@ -213,6 +214,11 @@ public:
* Setting batch_size to excessively large or excessively small values may impact negatively the
* performance.
*
* ### Threads
*
* When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the
* hood to do some lookahead.
*
* ### REQUIRED: Buffer Padding
*
* The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what
@@ -222,9 +228,6 @@ public:
*
* ### Threads
*
* When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the
* hood to do some lookahead.
*
* ### Parser Capacity
*
* If the parser's current capacity is less than batch_size, it will allocate enough capacity
@@ -249,6 +252,8 @@ public:
*/
inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
@@ -358,13 +363,39 @@ public:
bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept;
#endif
/**
* Get a unique parser instance corresponding to the current thread.
* This instance can be safely used within the current thread, but it should
* not be passed to other threads.
*
* A parser should only be used for one document at a time.
*
* Our simdjson::from functions use this parser instance.
*
* You can free the related parser by calling release_parser().
*/
static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser();
/**
* Release the parser instance initialized by get_parser() and all the
* associated resources (memory). Returns true if a parser instance
* was released.
*/
static simdjson_inline bool release_parser();
private:
friend bool release_parser();
friend ondemand::parser& get_parser();
/** Get the thread-local parser instance, allocates it if needed */
static simdjson_inline simdjson_warn_unused std::unique_ptr<ondemand::parser>& get_parser_instance();
/** Get the thread-local parser instance, it might be null */
static simdjson_inline simdjson_warn_unused std::unique_ptr<ondemand::parser>& get_threadlocal_parser_if_exists();
/** @private [for benchmarking access] The implementation to use */
std::unique_ptr<simdjson::internal::dom_parser_implementation> implementation{};
size_t _capacity{0};
size_t _max_capacity;
size_t _max_depth{DEFAULT_MAX_DEPTH};
std::unique_ptr<uint8_t[]> string_buf{};
#if SIMDJSON_DEVELOPMENT_CHECKS
std::unique_ptr<token_position[]> start_positions{};
#endif
+2 -2
View File
@@ -409,9 +409,9 @@ using std::operator<<;
#endif
#if nssv_HAVE_NODISCARD
# define nssv_nodiscard [[nodiscard]]
# define nssv_nodiscard simdjson_warn_unused
#else
# define nssv_nodiscard /*[[nodiscard]]*/
# define nssv_nodiscard /*simdjson_warn_unused*/
#endif
// Additional includes:
+3 -3
View File
@@ -1,4 +1,4 @@
/* auto-generated on 2025-08-05 16:29:59 +0000. version 4.0.0 Do not edit! */
/* auto-generated on 2025-08-12 19:38:50 -0400. version 4.0.0 Do not edit! */
/* including simdjson.cpp: */
/* begin file simdjson.cpp */
#define SIMDJSON_SRC_SIMDJSON_CPP
@@ -1039,9 +1039,9 @@ using std::operator<<;
#endif
#if nssv_HAVE_NODISCARD
# define nssv_nodiscard [[nodiscard]]
# define nssv_nodiscard simdjson_warn_unused
#else
# define nssv_nodiscard /*[[nodiscard]]*/
# define nssv_nodiscard /*simdjson_warn_unused*/
#endif
// Additional includes:
+1089 -496
View File
File diff suppressed because it is too large Load Diff
Binary file not shown.
+80 -76
View File
@@ -61,27 +61,6 @@ struct Car {
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>>,
"It should be deserializable");
static_assert(std::input_or_output_iterator<simdjson::auto_iterator>,
"Must be a valid input iterator");
static_assert(std::semiregular<simdjson::auto_iterator>,
"Should be kinda regular");
// static_assert(std::ranges::__access::__member_end<simdjson::auto_parser<>>,
// "Must be a valid input iterator");
// static_assert(std::ranges::views::__adaptor::__is_range_adaptor_closure<
// simdjson::auto_parser<>>,
// "Parser need to be range adaptor closure.");
// static_assert(std::ranges::views::__adaptor::__adaptor_invocable<
// decltype(simdjson::to<Car>()),
// simdjson::auto_parser<>>,
// "I don't even know!");
static_assert(std::ranges::range<simdjson::auto_parser<>>,
"Parser need to be a range.");
static_assert(std::ranges::forward_range<simdjson::auto_parser<>>,
"Parser need to be an input range.");
static_assert(
requires(simdjson::auto_parser<> &parser) {
{ parser.begin() } -> std::input_or_output_iterator;
}, "Must be valid iterator.");
simdjson::padded_string json_car =
R"( {
@@ -165,23 +144,6 @@ bool to_array() {
TEST_SUCCEED();
}
bool to_array_shortcut() {
TEST_START();
simdjson::ondemand::parser parser;
for (auto val : simdjson::from(parser, json_cars)) {
Car car{};
if (auto const error = val.get(car)) {
std::cerr << simdjson::error_message(error) << std::endl;
return false;
}
if (car.year < 1998) {
std::cerr << car.make << " " << car.model << " " << car.year << std::endl;
return false;
}
}
TEST_SUCCEED();
}
bool to_bad_array() {
TEST_START();
auto parser = simdjson::from(json_car);
@@ -223,43 +185,17 @@ bool to_bad_array() {
TEST_SUCCEED();
}
bool test_basic_adaptor() {
TEST_START();
for (Car car : simdjson::from(json_cars) | simdjson::as<Car>()) {
if (car.year < 1998) {
return false;
}
}
TEST_SUCCEED();
}
bool test_basic_adaptor_with_parser() {
#if SIMDJSON_SUPPORTS_RANGES
// currently not active (SIMDJSON_SUPPORTS_RANGES)
bool to_array_shortcut() {
TEST_START();
simdjson::ondemand::parser parser;
for (Car car : simdjson::from(parser, json_cars) | simdjson::as<Car>()) {
if (car.year < 1998) {
for (auto val : simdjson::from(parser, json_cars)) {
Car car{};
if (auto const error = val.get(car)) {
std::cerr << simdjson::error_message(error) << std::endl;
return false;
}
}
TEST_SUCCEED();
}
bool test_no_errors() {
TEST_START();
auto cars = simdjson::from(json_cars) | simdjson::no_errors;
for (auto val : cars) {
Car car = val.get<Car>();
if (car.year < 1998) {
return false;
}
}
TEST_SUCCEED();
}
bool to_clean_array() {
TEST_START();
for (auto val : simdjson::from(json_cars) | simdjson::no_errors) {
Car car = val.get<Car>();
if (car.year < 1998) {
std::cerr << car.make << " " << car.model << " " << car.year << std::endl;
return false;
@@ -267,7 +203,70 @@ bool to_clean_array() {
}
TEST_SUCCEED();
}
// currently not active (SIMDJSON_SUPPORTS_RANGES)
bool test_basic_adaptor() {
TEST_START();
int64_t sum_year = 0;
for (Car car : simdjson::from(json_cars) | simdjson::as<Car>()) {
printf("Car: %s %s %d\n", car.make.c_str(), car.model.c_str(), car.year);
if (car.year < 1998) {
return false;
}
sum_year += car.year;
}
ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999);
TEST_SUCCEED();
}
// currently not active (SIMDJSON_SUPPORTS_RANGES)
bool test_basic_adaptor_with_parser() {
TEST_START();
simdjson::ondemand::parser parser;
int64_t sum_year = 0;
for (Car car : simdjson::from(parser, json_cars) | simdjson::as<Car>()) {
if (car.year < 1998) {
return false;
}
sum_year += car.year;
}
ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999);
TEST_SUCCEED();
}
// currently not active (SIMDJSON_SUPPORTS_RANGES)
bool test_no_errors() {
TEST_START();
int64_t sum_year = 0;
auto cars = simdjson::from(json_cars) | simdjson::no_errors;
for (auto val : cars) {
Car car = val.get<Car>();
if (car.year < 1998) {
return false;
}
printf("-- year: %d\n", car.year);
sum_year += car.year;
}
ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999);
TEST_SUCCEED();
}
// currently not active (SIMDJSON_SUPPORTS_RANGES)
bool to_clean_array() {
TEST_START();
int64_t sum_year = 0;
for (auto val : simdjson::from(json_cars) | simdjson::no_errors) {
Car car = val.get<Car>();
if (car.year < 1998) {
std::cerr << car.make << " " << car.model << " " << car.year << std::endl;
return false;
}
sum_year += car.year;
}
ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999);
TEST_SUCCEED();
}
// currently not active (SIMDJSON_SUPPORTS_RANGES)
bool test_to_adaptor_basic() {
TEST_START();
// Test 1: Basic usage of to<T> with a value reference
@@ -287,6 +286,8 @@ bool test_to_adaptor_basic() {
TEST_SUCCEED();
}
#endif // SIMDJSON_SUPPORTS_RANGES
bool test_to_adaptor_with_single_value() {
TEST_START();
// Test 2: Using to<T> to convert individual values
@@ -338,12 +339,15 @@ bool test_to_vs_from_equivalence() {
bool run() {
return
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION
test_basic_adaptor() && broken() && simple() && simple_optional() && with_parser() && to_array() &&
to_array_shortcut() && to_bad_array() && test_no_errors() &&
to_clean_array() && test_to_adaptor_basic() &&
broken() && simple() && simple_optional() && with_parser() && to_array() &&
to_bad_array() &&
test_to_adaptor_with_single_value() && test_to_vs_from_equivalence() &&
test_basic_adaptor_with_parser() && example_with_parser() &&
#endif // SIMDJSON_EXCEPTIONS
example_with_parser() &&
#if SIMDJSON_SUPPORTS_RANGES
test_basic_adaptor_with_parser() && test_no_errors() && test_basic_adaptor()
&& test_to_adaptor_basic() && to_clean_array() && to_array_shortcut() &&
#endif // SIMDJSON_SUPPORTS_RANGES
#endif // SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION
true;
}