diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index 29a305d51..4fc8518e4 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -37,4 +37,11 @@ endif() if(SIMDJSON_STATIC_REFLECTION) add_subdirectory(static_reflect) -endif(SIMDJSON_STATIC_REFLECTION) \ No newline at end of file +endif(SIMDJSON_STATIC_REFLECTION) + + +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-std=c++20" SIMDJSON_COMPILER_SUPPORTS_CXX20) +if(SIMDJSON_EXCEPTIONS AND SIMDJSON_COMPILER_SUPPORTS_CXX20) + add_subdirectory(from) +endif() \ No newline at end of file diff --git a/benchmark/from/CMakeLists.txt b/benchmark/from/CMakeLists.txt new file mode 100644 index 000000000..67e25524d --- /dev/null +++ b/benchmark/from/CMakeLists.txt @@ -0,0 +1,14 @@ +# Executable +add_executable(from_benchmark from_benchmark.cpp) + +# Compile for C++20. +target_compile_features(from_benchmark PRIVATE cxx_std_20) + +# Check if -march=native is supported +include(CheckCXXCompilerFlag) +check_cxx_compiler_flag("-march=native" SIMDJSON_SUPPORTS_MARCH_NATIVE) +if(SIMDJSON_SUPPORTS_MARCH_NATIVE) + target_compile_options(from_benchmark PRIVATE -march=native) +endif() + +target_include_directories(from_benchmark PRIVATE ${CMAKE_CURRENT_LIST_DIR}/..) \ No newline at end of file diff --git a/benchmark/from/benchmark_helpers.h b/benchmark/from/benchmark_helpers.h new file mode 100644 index 000000000..b6273b1d1 --- /dev/null +++ b/benchmark/from/benchmark_helpers.h @@ -0,0 +1,77 @@ +#ifndef BENCHMARK_HELPERS_H +#define BENCHMARK_HELPERS_H + +#include "event_counter.h" +#include + +event_collector collector; + +template +std::pair +bench(const function_type &&function, size_t min_repeat = 10, + size_t min_time_ns = 40'000'000, size_t max_repeat = 10000000) { + size_t N = min_repeat; + if (N == 0) { + N = 1; + } + event_aggregate warm_aggregate{}; + for (size_t i = 0; i < N; i++) { + std::atomic_thread_fence(std::memory_order_acquire); + collector.start(); + function(); + std::atomic_thread_fence(std::memory_order_release); + event_count allocate_count = collector.end(); + warm_aggregate << allocate_count; + if ((i + 1 == N) && (warm_aggregate.total_elapsed_ns() < min_time_ns) && + (N < max_repeat)) { + N *= 10; + } + } + event_aggregate aggregate{}; + for (size_t i = 0; i < 10; i++) { + std::atomic_thread_fence(std::memory_order_acquire); + collector.start(); + for (size_t i = 0; i < N; i++) { + function(); + } + std::atomic_thread_fence(std::memory_order_release); + event_count allocate_count = collector.end(); + aggregate << allocate_count; + } + return {aggregate, N}; +} + +double pretty_print(const std::string &name, size_t num_chars, + std::pair result) { + const auto &agg = result.first; + size_t N = result.second; + num_chars *= N; + printf("%-40s : %8.2f ns %8.2f GB/s", name.c_str(), + agg.elapsed_ns() / num_chars, num_chars / agg.elapsed_ns()); + if (collector.has_events()) { + printf(" %8.2f GHz %8.2f cycles/char %8.2f ins./char %8.2f i/c", + agg.cycles() / agg.elapsed_ns(), agg.cycles() / num_chars, + agg.instructions() / num_chars, agg.instructions() / agg.cycles()); + } + printf("\n"); + return num_chars / agg.elapsed_ns(); +} + +double pretty_print_array(const std::string &name, size_t num_chars, size_t num_elements, + std::pair result) { + const auto &agg = result.first; + size_t N = result.second; + num_chars *= N; + printf("%-40s : %8.2f ns/char %8.2f ns/value %8.2f GB/s", name.c_str(), + agg.elapsed_ns() / num_chars, agg.elapsed_ns() / num_elements, num_chars / agg.elapsed_ns()); + if (collector.has_events()) { + printf(" %8.2f GHz %8.2f cycles/char %8.2f ins./char %8.2f ins./value %8.2f i/c", + agg.cycles() / agg.elapsed_ns(), agg.cycles() / num_chars, + agg.instructions() / num_chars, agg.instructions() / num_elements, agg.instructions() / agg.cycles()); + } + printf("\n"); + return num_chars / agg.elapsed_ns(); +} + + +#endif \ No newline at end of file diff --git a/benchmark/from/car_helpers.h b/benchmark/from/car_helpers.h new file mode 100644 index 000000000..c77f5b00b --- /dev/null +++ b/benchmark/from/car_helpers.h @@ -0,0 +1,146 @@ +#ifndef CAR_HELPERS_H +#define CAR_HELPERS_H + +#include +#include +#include +#include + +#include + +struct Car { + std::string make; + std::string model; + int64_t year; // We deliberately do not include the tire pressure. +}; + +using namespace simdjson; + +std::string random_car_json() { + static const std::vector makes = {"Toyota", "Honda", "Ford", + "BMW", "Mazda"}; + static const std::vector models = {"Camry", "Civic", "Focus", + "320i", "3"}; + static thread_local std::mt19937 rng{std::random_device{}()}; + std::uniform_int_distribution make_dist(0, makes.size() - 1); + std::uniform_int_distribution model_dist(0, models.size() - 1); + std::uniform_int_distribution year_dist(2000, 2025); + std::uniform_real_distribution pressure_dist(30.0, 45.0); + + std::ostringstream oss; + oss << R"({ "make": ")" << makes[make_dist(rng)] << R"(", "model": ")" + << models[model_dist(rng)] << R"(", "year": )" << year_dist(rng) + << R"(, "tire_pressure": [ )" << std::fixed << std::setprecision(1) + << pressure_dist(rng) << ", " << pressure_dist(rng) << R"( ] })"; + return oss.str(); +} + +std::string generate_car_json_stream(size_t N) { + std::ostringstream oss; + for (size_t i = 0; i < N; ++i) { + oss << random_car_json(); + oss << "\n"; + } + return oss.str(); +} + +std::string generate_car_json_array(size_t N) { + std::ostringstream oss; + oss << "[\n"; + for (size_t i = 0; i < N; ++i) { + oss << random_car_json(); + if (i < N - 1) { + oss << ","; + } + oss << "\n"; + } + oss << "]"; + return oss.str(); +} + +// We want to maximize C++ portability, but this is not needed with static +// reflection (C++26). +template <> +simdjson_inline simdjson_result +simdjson::ondemand::document::get() & noexcept { + ondemand::object obj; + auto error = get_object().get(obj); + if (error) { + return error; + } + Car car; + if ((error = obj["make"].get_string(car.make))) { + return error; + } + if ((error = obj["model"].get_string(car.model))) { + return error; + } + if ((error = obj["year"].get_int64().get(car.year))) { + return error; + } + return car; +} + +template <> +simdjson_inline simdjson_result simdjson::ondemand::value::get() noexcept { + ondemand::object obj; + auto error = get_object().get(obj); + if (error) { + return error; + } + Car car; + if ((error = obj["make"].get_string(car.make))) { + return error; + } + if ((error = obj["model"].get_string(car.model))) { + return error; + } + if ((error = obj["year"].get_int64().get(car.year))) { + return error; + } + return car; +} + +template <> +simdjson_inline simdjson_result +simdjson::ondemand::document_reference::get() & noexcept { + ondemand::object obj; + auto error = get_object().get(obj); + if (error) { + return error; + } + Car car; + if ((error = obj["make"].get_string(car.make))) { + return error; + } + if ((error = obj["model"].get_string(car.model))) { + return error; + } + if ((error = obj["year"].get_int64().get(car.year))) { + return error; + } + return car; +} + +template <> +simdjson_inline simdjson_result +simdjson::ondemand::document_reference::get() && noexcept { + ondemand::object obj; + auto error = get_object().get(obj); + if (error) { + return error; + } + Car car; + if ((error = obj["make"].get_string(car.make))) { + return error; + } + if ((error = obj["model"].get_string(car.model))) { + return error; + } + if ((error = obj["year"].get_int64().get(car.year))) { + return error; + } + return car; +} + +#endif \ No newline at end of file diff --git a/benchmark/from/from_benchmark.cpp b/benchmark/from/from_benchmark.cpp new file mode 100644 index 000000000..819868dd0 --- /dev/null +++ b/benchmark/from/from_benchmark.cpp @@ -0,0 +1,132 @@ +// We are going to assume C++20. + +#include +#include +#include +#include +#include + +#include "benchmark_helpers.h" +#include "car_helpers.h" + +using namespace simdjson; + +void run_benchmarks() { + printf("Running benchmarks on tiny input...\n"); + simdjson::padded_string json = + R"({ "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9 ] })"_padded; + volatile size_t dummy = 0; + ondemand::parser parser; + + // Classic On-Demand + pretty_print("simdjson classic", json.size(), + bench([&json, &dummy, &parser]() { + ondemand::document doc = parser.iterate(json); + Car car = doc.get(); + dummy = dummy + car.year; + })); + + // Benchmark simdjson::from() without parser + pretty_print("simdjson::from() (no parser)", json.size(), + bench([&json, &dummy]() { + Car car = simdjson::from(json); + dummy = dummy + car.year; + })); + + // Benchmark simdjson::from() with parser + pretty_print("simdjson::from() (with parser)", json.size(), + bench([&json, &dummy, &parser]() { + Car car = simdjson::from(parser, json); + dummy = dummy + car.year; + })); +} + +void run_array_benchmarks() { + printf("Running benchmarks on large array...\n"); + size_t N = 1'000'000; + simdjson::padded_string json = generate_car_json_array(N); + volatile size_t dummy = 0; + ondemand::parser parser; + + // Classic On-Demand + pretty_print_array("simdjson classic", json.size(), N, + bench([&json, &dummy, &parser]() { + dummy = 0; + ondemand::document doc = parser.iterate(json); + ondemand::array array = doc.get_array(); + for (auto value : array) { + Car car = value.get(); + dummy = dummy + car.year; + } + })); + + // from with array + pretty_print_array("simdjson::from(json).array()", json.size(), N, + bench([&json, &dummy, &parser]() { + dummy = 0; + + for (auto value : simdjson::from(json).array()) { + Car car = value.get(); + dummy = dummy + car.year; + } + })); +#if SIMDJSON_SUPPORTS_RANGES + // Benchmark simdjson::from() without parser (EXPERIMENTAL) + pretty_print_array("simdjson::from() (experimental, no parser)", + json.size(), N, bench([&json, &dummy]() { + dummy = 0; + for (Car car : + simdjson::from(json) | simdjson::as()) { + dummy = dummy + car.year; + } + })); + + // Benchmark simdjson::from() with parser (EXPERIMENTAL) + pretty_print_array("simdjson::from() (experimental, with parser)", + json.size(), N, bench([&json, &dummy, &parser]() { + dummy = 0; + for (Car car : simdjson::from(parser, json) | + simdjson::as()) { + dummy = dummy + car.year; + } + })); +#endif // SIMDJSON_SUPPORTS_RANGES +} + +void run_stream_benchmarks() { + printf("Running stream benchmarks...\n"); + simdjson::padded_string json = generate_car_json_stream(1'000'000); + volatile size_t dummy = 0; + ondemand::parser parser; + + // Classic On-Demand + pretty_print("simdjson classic", json.size(), + bench([&json, &dummy, &parser]() { + ondemand::document_stream stream = parser.iterate_many(json); + for (auto doc : stream) { + Car car = doc.get(); + dummy = dummy + car.year; + } + })); + // from_many + pretty_print("simdjson with thread local", json.size(), + bench([&json, &dummy, &parser]() { + ondemand::document_stream stream = + ondemand::parser::get_parser().iterate_many(json); + for (auto doc : stream) { + Car car = doc.get(); + dummy = dummy + car.year; + } + })); +} + +int main() { + for (size_t trial = 0; trial < 3; trial++) { + printf("Trial %zu:\n", trial + 1); + run_array_benchmarks(); + run_stream_benchmarks(); + run_benchmarks(); + printf("\n"); + } + return EXIT_SUCCESS; +} diff --git a/doc/basics.md b/doc/basics.md index 3d17a482f..4b389ab35 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -180,6 +180,18 @@ auto json = padded_string::load("twitter.json"); // load JSON file 'twitter.json ondemand::document doc = parser.iterate(json); // position a pointer at the beginning of the JSON data ``` +If you prefer not to create your own `ondemand::parser` instance, you can access +a thread-local version by calling `ondemand::parser.get_parser()`. + + +```c++ +ondemand::document doc = ondemand::parser.get_parser().iterate(json); +``` + +However, you should be careful because a parser instance can only be used for one +document at a time, thus it is only applicable when you are only parsing one +document per thread at any one time. + You can also create a padded string---and call `iterate()`: ```c++ @@ -1351,57 +1363,25 @@ that are not made by Toyota. For even more convenience, you can do it directly without a document instance like so: ```cpp - simdjson::ondemand::parser parser; - Car car = simdjson::from(parser, json_car); +Car car = simdjson::from(json); ``` -We strongly encourage you to rely on an parser instance (`simdjson::ondemand::parser`) which you reuse. However, if performance is not a concern, you can omit the declaration -of a parser instance like so: +You can also use the `simdjson::from` syntax to iterate over an array. ```cpp -Car car = simdjson::from(json); +for(auto val : simdjson::from(json).array()) { + Car c = val.get(); // ... +} ``` Standard STL types are supported: ```cpp -simdjson::ondemand::parser parser; std::map obj = -simdjson::from(parser, R"({"key": "value"})"_padded); + simdjson::from(R"({"key": "value"})"_padded); ``` -You can also use C++20 ranges to iterate over an array: - -```cpp - simdjson::padded_string json_cars = - R"( [ { "make": "Toyota", "model": "Camry", "year": 2018, - "tire_pressure": [ 40.1, 39.9 ] }, - { "make": "Kia", "model": "Soul", "year": 2012, - "tire_pressure": [ 30.1, 31.0 ] }, - { "make": "Toyota", "model": "Tercel", "year": 1999, - "tire_pressure": [ 29.8, 30.0 ] } - ])"_padded; - - simdjson::ondemand::parser parser; - for (Car car : simdjson::from(parser, json_cars) | simdjson::as()) { - if (car.year < 1998) { - return false; - } - } -``` - -Again, if performance is not a concern, you can omit the parser instance. - -```cpp - for (Car car : simdjson::from(json_cars) | simdjson::as()) { - if (car.year < 1998) { - return false; - } - } -``` - - ### 3. Using static reflection (C++26) If you have a C++26 compatible compiler, you can compile @@ -1426,33 +1406,17 @@ Car c = doc.get(); Just like when using `tag_invoke` for custom types (but without the `tag_invoke` code), you can parse a class instance directly without a parser instance: ```cpp -simdjson::ondemand::parser parser; -Car car = simdjson::from(parser, json); +Car car = simdjson::from(json); ``` -Similarly, you can also use C++20 ranges to iterate over an array: +You can also use the `simdjson::from` syntax to iterate over an array. ```cpp - simdjson::padded_string json_cars = - R"( [ { "make": "Toyota", "model": "Camry", "year": 2018, - "tire_pressure": [ 40.1, 39.9 ] }, - { "make": "Kia", "model": "Soul", "year": 2012, - "tire_pressure": [ 30.1, 31.0 ] }, - { "make": "Toyota", "model": "Tercel", "year": 1999, - "tire_pressure": [ 29.8, 30.0 ] } - ])"_padded; - simdjson::ondemand::parser parser; - for (Car car : simdjson::from(, parserjson_cars) | simdjson::as()) { - if (car.year < 1998) { - return false; - } - } +for(auto val : simdjson::from(json).array()) { + Car c = val.get(); // ... +} ``` -When using the `simdjson::from` syntax, you can also omit the parser instance, -for more convenience. However, we strongly encourage you to create a single parser -instance that is reused, as it leads to better performance. - You can also automatically serialize the `Car` instance to a JSON string, see our [Builder documentation](builder.md). diff --git a/include/simdjson/compiler_check.h b/include/simdjson/compiler_check.h index 8ce7ab697..0d3321507 100644 --- a/include/simdjson/compiler_check.h +++ b/include/simdjson/compiler_check.h @@ -94,11 +94,11 @@ #endif // defined(__cpp_concepts) && !defined(SIMDJSON_CONCEPT_DISABLED) #if !defined(SIMDJSON_CONSTEVAL) -#if defined(__cpp_consteval) && __cpp_consteval >= 201811L +#if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #define SIMDJSON_CONSTEVAL 1 #else #define SIMDJSON_CONSTEVAL 0 -#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L +#endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #endif // !defined(SIMDJSON_CONSTEVAL) #endif // SIMDJSON_COMPILER_CHECK_H diff --git a/include/simdjson/convert.h b/include/simdjson/convert.h index 6e8448d47..2e9616398 100644 --- a/include/simdjson/convert.h +++ b/include/simdjson/convert.h @@ -4,76 +4,11 @@ #include "simdjson/ondemand.h" #include -#ifdef __cpp_lib_ranges -#include -#endif namespace simdjson { -struct [[nodiscard]] auto_iterator_end {}; - -/** - * A Wrapper for simdjson_result in order to make it - * compatible with ranges (to satisfy std::ranges::input_range). - */ -struct [[nodiscard]] auto_iterator { - using iterator_category = std::forward_iterator_tag; - using type = simdjson_result; - using value_type = simdjson_result; // type::value_type - using reference = value_type &; - using const_reference = const value_type &; - using difference_type = std::ptrdiff_t; - - struct auto_iterator_storage { - type m_iter{}; - mutable value_type m_value{}; - }; - -private: - auto_iterator_storage *m_storage = nullptr; - -public: - constexpr auto_iterator() noexcept = default; - explicit auto_iterator(auto_iterator_storage &storage) noexcept - : m_storage{&storage} {}; - auto_iterator(auto_iterator const &) = default; - auto_iterator(auto_iterator &&) = default; - auto_iterator &operator=(auto_iterator const &) = default; - auto_iterator &operator=(auto_iterator &&) noexcept = default; - ~auto_iterator() = default; - - reference operator*() const noexcept { return m_storage->m_value; } - reference operator*() noexcept { return m_storage->m_value; } - - auto_iterator &operator++() noexcept { - ++m_storage->m_iter; - m_storage->m_value = - m_storage->m_iter.at_end() || m_storage->m_iter.error() != SUCCESS - ? value_type{} - : *m_storage->m_iter; - return *this; - } - auto_iterator operator++(int) noexcept { - auto_iterator const tmp = *this; - operator++(); - return tmp; - } - - [[nodiscard]] bool operator==(auto_iterator const &other) const noexcept { - return m_storage == other.m_storage && - m_storage->m_iter == other.m_storage->m_iter; - } - - [[nodiscard]] bool operator==(auto_iterator_end) const noexcept { - return m_storage != nullptr && m_storage->m_iter.at_end(); - } -}; - -template -struct [[nodiscard]] auto_parser -#if __cpp_lib_ranges - : std::ranges::view_interface> -#endif +template +struct auto_parser { using value_type = simdjson_result; using size_type = size_t; @@ -82,17 +17,12 @@ struct [[nodiscard]] auto_parser using const_pointer = const value_type *; using reference = value_type &; using const_reference = const value_type &; - using iterator = auto_iterator; - using const_iterator = auto_iterator; // auto_iterator is already const private: ParserType m_parser; ondemand::document m_doc; error_code m_error{SUCCESS}; - // Caching the iterator here: - iterator::auto_iterator_storage iter_storage{}; - template static constexpr bool is_nothrow_gettable = requires(ondemand::document doc) { { doc.get() } noexcept; @@ -111,10 +41,6 @@ public: m_error = m_parser.iterate(str).get(m_doc); } - explicit auto_parser(padded_string_view const str) noexcept - requires(!std::is_pointer_v) - : auto_parser{ParserType{}, str} {} - // pointer constructors: explicit auto_parser(std::remove_pointer_t &parser, ondemand::document &&doc) noexcept @@ -128,6 +54,10 @@ public: m_error = m_parser->iterate(str).get(m_doc); } + explicit auto_parser(padded_string_view const str) noexcept + requires(std::is_pointer_v) + : auto_parser{ondemand::parser::get_parser(), str} {} + explicit auto_parser(ParserType parser, ondemand::document &&doc) noexcept requires(std::is_pointer_v) : auto_parser{*parser, std::move(doc)} {} @@ -139,7 +69,7 @@ public: ~auto_parser() = default; /// Get the parser - [[nodiscard]] std::remove_pointer_t &parser() noexcept { + simdjson_warn_unused std::remove_pointer_t &parser() noexcept { if constexpr (std::is_pointer_v) { return *m_parser; } else { @@ -148,7 +78,7 @@ public: } template - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result result() noexcept(is_nothrow_gettable) { if (m_error != SUCCESS) { return m_error; @@ -157,29 +87,29 @@ public: return m_doc.get(); } - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result array() noexcept { return result(); } - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result object() noexcept { return result(); } - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result number() noexcept { return result(); } template - [[nodiscard]] simdjson_inline explicit(false) + simdjson_warn_unused simdjson_inline explicit(false) operator simdjson_result() noexcept(is_nothrow_gettable) { return result(); } template - [[nodiscard]] simdjson_inline explicit(false) operator T() noexcept(false) { + simdjson_warn_unused simdjson_inline explicit(false) operator T() noexcept(false) { if (m_error != SUCCESS) { throw simdjson_error(m_error); } @@ -192,7 +122,7 @@ public: // We also cannot have "operator T&" without manual memory management either. template - [[nodiscard]] simdjson_inline std::optional + simdjson_warn_unused simdjson_inline std::optional optional() noexcept(is_nothrow_gettable) { if (m_error != SUCCESS) { return std::nullopt; @@ -204,76 +134,25 @@ public: } return {std::move(value)}; } - - simdjson_inline auto_iterator begin() noexcept { - if (m_error != SUCCESS) { - // Create an iterator with the error - iter_storage.m_iter = iterator::type(m_error); - iter_storage.m_value = value_type{}; - return auto_iterator{iter_storage}; - } - if (iter_storage.m_iter.error() != SUCCESS && - !iter_storage.m_iter.at_end()) { - // Try to get the document as an array - ondemand::array arr; - if(auto error = m_doc.get_array().get(arr); error == SUCCESS) { - iter_storage = {.m_iter = iterator::type{arr.begin()}, - .m_value = iterator::value_type{ - iter_storage.m_iter.at_end() || - iter_storage.m_iter.error() != SUCCESS - ? value_type{} - : *iter_storage.m_iter}}; - } else { - // If it's not an array, create an error iterator - iter_storage.m_iter = iterator::type(error); - iter_storage.m_value = value_type{}; - } - } - return auto_iterator{iter_storage}; - } - simdjson_inline auto_iterator_end end() noexcept { return {}; } }; -#ifdef __cpp_lib_ranges -// For C++20, we implement our own pipe operator since range_adaptor_closure is C++23 -static constexpr struct [[nodiscard]] no_errors_adaptor { - - [[nodiscard]] bool - operator()(simdjson_result const &val) const noexcept { - return val.error() == SUCCESS; - } - - template - auto operator()(Range &&rng) const noexcept { - return std::forward(rng) | std::views::filter(*this); - } -} no_errors; template -struct [[nodiscard]] to_adaptor { +struct to_adaptor { /// Convert to T - [[nodiscard]] T - operator()(simdjson_result &val) const noexcept { + T operator()(simdjson_result &val) const noexcept { return val.get(); } - - /// Make it an adaptor - template - auto operator()(Range &&rng) const noexcept { - return std::forward(rng) | no_errors | std::views::transform(*this); - } - /** * Parse input string into any object if possible. This function call * will return an auto_parser that can be used to extract the desired * object from the input string. The ondemand::parser instance is created * internally. * - * *WARNING*: This function call will create a new parser instance each time - * it is called. This has performance implications. We strongly recommend - * that you create a parser instance once and reuse it many times. + * This function uses the simdjson::ondemand::parser::get_parser() instance. + * A parser should only be used for one document at a time. */ auto operator()(padded_string_view const str) const noexcept { return auto_parser{str}; @@ -300,34 +179,26 @@ template static constexpr to_adaptor to{}; * Example usage: * * ```cpp + * std::map obj = + * simdjson::from(R"({"key": "value"})"_padded); + * ``` + * + * This will parse the JSON string and return an object representation. By default, we + * use the simdjson::ondemand::parser::get_parser() instance. A parser instance should + * be used for just one document at a time. + * + * You can also pass you own parser instance: + * ```cpp * simdjson::ondemand::parser parser; * std::map obj = * simdjson::from(parser, R"({"key": "value"})"_padded); * ``` + * The parser instance can be reused. * - * This will parse the JSON string and return an object representation. The `ondemand::parser` - * instance can be reused. It is also possible to omit the parser instance, in which case a parser - * instance will be created internally, although this can have negative performance consequences. */ static constexpr to_adaptor<> from{}; -template -using as = to_adaptor; - -// For C++20 ranges without range_adaptor_closure, we need to define pipe operators -template -inline auto operator|(Range&& range, const no_errors_adaptor& adaptor) { - return adaptor(std::forward(range)); -} - -template -inline auto operator|(Range&& range, const to_adaptor& adaptor) { - return adaptor(std::forward(range)); -} - -#endif // __cpp_lib_ranges - } // namespace simdjson #endif // __cpp_concepts -#endif // SIMDJSON_CONVERT_H +#endif // SIMDJSON_CONVERT_H \ No newline at end of file diff --git a/include/simdjson/generic/ondemand/array_iterator.h b/include/simdjson/generic/ondemand/array_iterator.h index 4a0bbb84a..2e0d20c9f 100644 --- a/include/simdjson/generic/ondemand/array_iterator.h +++ b/include/simdjson/generic/ondemand/array_iterator.h @@ -62,7 +62,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -95,7 +95,7 @@ struct simdjson_result : publ simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson diff --git a/include/simdjson/generic/ondemand/deserialize.h b/include/simdjson/generic/ondemand/deserialize.h index 1e310fba6..d34674a88 100644 --- a/include/simdjson/generic/ondemand/deserialize.h +++ b/include/simdjson/generic/ondemand/deserialize.h @@ -97,35 +97,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } diff --git a/include/simdjson/generic/ondemand/document_stream-inl.h b/include/simdjson/generic/ondemand/document_stream-inl.h index 0dd10feac..136ca7994 100644 --- a/include/simdjson/generic/ondemand/document_stream-inl.h +++ b/include/simdjson/generic/ondemand/document_stream-inl.h @@ -182,10 +182,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. diff --git a/include/simdjson/generic/ondemand/document_stream.h b/include/simdjson/generic/ondemand/document_stream.h index ef1265fb3..8b91d0c0f 100644 --- a/include/simdjson/generic/ondemand/document_stream.h +++ b/include/simdjson/generic/ondemand/document_stream.h @@ -131,6 +131,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -144,6 +145,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -187,6 +189,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -198,6 +205,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. diff --git a/include/simdjson/generic/ondemand/parser-inl.h b/include/simdjson/generic/ondemand/parser-inl.h index 8350fe738..d1e386940 100644 --- a/include/simdjson/generic/ondemand/parser-inl.h +++ b/include/simdjson/generic/ondemand/parser-inl.h @@ -145,6 +145,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -189,6 +196,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson diff --git a/include/simdjson/generic/ondemand/parser.h b/include/simdjson/generic/ondemand/parser.h index 5e22f5554..0bffe5337 100644 --- a/include/simdjson/generic/ondemand/parser.h +++ b/include/simdjson/generic/ondemand/parser.h @@ -7,6 +7,7 @@ #endif // SIMDJSON_CONDITIONAL_INCLUDE #include +#include namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { @@ -213,6 +214,11 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * * ### REQUIRED: Buffer Padding * * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what @@ -222,9 +228,6 @@ public: * * ### Threads * - * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the - * hood to do some lookahead. - * * ### Parser Capacity * * If the parser's current capacity is less than batch_size, it will allocate enough capacity @@ -249,6 +252,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -358,13 +363,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif diff --git a/include/simdjson/nonstd/string_view.hpp b/include/simdjson/nonstd/string_view.hpp index 6ee1c01ce..2fbe868df 100644 --- a/include/simdjson/nonstd/string_view.hpp +++ b/include/simdjson/nonstd/string_view.hpp @@ -409,9 +409,9 @@ using std::operator<<; #endif #if nssv_HAVE_NODISCARD -# define nssv_nodiscard [[nodiscard]] +# define nssv_nodiscard simdjson_warn_unused #else -# define nssv_nodiscard /*[[nodiscard]]*/ +# define nssv_nodiscard /*simdjson_warn_unused*/ #endif // Additional includes: diff --git a/singleheader/simdjson.cpp b/singleheader/simdjson.cpp index 05540f5ee..16fcd0059 100644 --- a/singleheader/simdjson.cpp +++ b/singleheader/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2025-08-05 16:29:59 +0000. version 4.0.0 Do not edit! */ +/* auto-generated on 2025-08-12 19:38:50 -0400. version 4.0.0 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -1039,9 +1039,9 @@ using std::operator<<; #endif #if nssv_HAVE_NODISCARD -# define nssv_nodiscard [[nodiscard]] +# define nssv_nodiscard simdjson_warn_unused #else -# define nssv_nodiscard /*[[nodiscard]]*/ +# define nssv_nodiscard /*simdjson_warn_unused*/ #endif // Additional includes: diff --git a/singleheader/simdjson.h b/singleheader/simdjson.h index 4ec81048b..f984cae90 100644 --- a/singleheader/simdjson.h +++ b/singleheader/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2025-08-05 16:29:59 +0000. version 4.0.0 Do not edit! */ +/* auto-generated on 2025-08-12 19:38:50 -0400. version 4.0.0 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -1059,9 +1059,9 @@ using std::operator<<; #endif #if nssv_HAVE_NODISCARD -# define nssv_nodiscard [[nodiscard]] +# define nssv_nodiscard simdjson_warn_unused #else -# define nssv_nodiscard /*[[nodiscard]]*/ +# define nssv_nodiscard /*simdjson_warn_unused*/ #endif // Additional includes: @@ -6641,8 +6641,6 @@ inline constexpr bool enable_view - namespace simdjson { /** @@ -6652,8 +6650,7 @@ namespace simdjson { */ namespace internal { -template -class base_formatter { +template class base_formatter { public: /** Add a comma **/ simdjson_inline void comma(); @@ -6692,24 +6689,76 @@ public: /** Prints one character **/ simdjson_inline void one_char(char c); + /** Prints characters in [begin, end) verbatim. **/ + simdjson_inline void chars(const char *begin, const char *end); + simdjson_inline void call_print_newline() { - static_cast(this)->print_newline(); + static_cast(this)->print_newline(); } simdjson_inline void call_print_indents(size_t depth) { - static_cast(this)->print_indents(depth); + static_cast(this)->print_indents(depth); } simdjson_inline void call_print_space() { - static_cast(this)->print_space(); + static_cast(this)->print_space(); } protected: // implementation details (subject to change) /** Backing buffer **/ - std::vector buffer{}; // not ideal! -}; + struct vector_with_small_buffer { + vector_with_small_buffer() = default; + ~vector_with_small_buffer() { free_buffer(); } + vector_with_small_buffer(const vector_with_small_buffer &) = delete; + vector_with_small_buffer & + operator=(const vector_with_small_buffer &) = delete; + + void clear() { + size = 0; + capacity = StaticCapacity; + free_buffer(); + buffer = array; + } + + simdjson_inline void push_back(char c) { + if (capacity < size + 1) + grow(capacity * 2); + buffer[size++] = c; + } + + simdjson_inline void append(const char *begin, const char *end) { + const size_t new_size = size + (end - begin); + if (capacity < new_size) + // std::max(new_size, capacity * 2); is broken in tests on Windows + grow(new_size < capacity * 2 ? capacity * 2 : new_size); + std::copy(begin, end, buffer + size); + size = new_size; + } + + std::string_view str() const { return std::string_view(buffer, size); } + + private: + void free_buffer() { + if (buffer != array) + delete[] buffer; + } + void grow(size_t new_capacity) { + auto new_buffer = new char[new_capacity]; + std::copy(buffer, buffer + size, new_buffer); + free_buffer(); + buffer = new_buffer; + capacity = new_capacity; + } + + static const size_t StaticCapacity = 64; + char array[StaticCapacity]; + char *buffer = array; + size_t size = 0; + size_t capacity = StaticCapacity; + } buffer{}; +}; /** * @private This is the class that we expect to use with the string_builder @@ -6747,8 +6796,7 @@ protected: * This is not to be confused with the simdjson::builder::string_builder * which is a different class. */ -template -class string_builder { +template class string_builder { public: /** Construct an initially empty builder, would print the empty string **/ string_builder() = default; @@ -6770,11 +6818,12 @@ public: simdjson_inline std::string_view str() const; /** Append a key_value_pair to the builder (to be printed) **/ simdjson_inline void append(simdjson::dom::key_value_pair value); + private: formatter format{}; }; -} // internal +} // namespace internal namespace dom { @@ -6783,33 +6832,43 @@ namespace dom { * * @param out The output stream. * @param value The element. - * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + * @throw if there is an error with the underlying output stream. simdjson + * itself will not throw. */ -inline std::ostream& operator<<(std::ostream& out, simdjson::dom::element value); +inline std::ostream &operator<<(std::ostream &out, + simdjson::dom::element value); #if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x); #endif /** * Print JSON to an output stream. * * @param out The output stream. * @param value The array. - * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + * @throw if there is an error with the underlying output stream. simdjson + * itself will not throw. */ -inline std::ostream& operator<<(std::ostream& out, simdjson::dom::array value); +inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value); #if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x); #endif /** * Print JSON to an output stream. * * @param out The output stream. * @param value The object. - * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + * @throw if there is an error with the underlying output stream. simdjson + * itself will not throw. */ -inline std::ostream& operator<<(std::ostream& out, simdjson::dom::object value); +inline std::ostream &operator<<(std::ostream &out, simdjson::dom::object value); #if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x); #endif } // namespace dom @@ -6821,47 +6880,47 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result -std::string to_string(T x) { - // in C++, to_string is standard: http://www.cplusplus.com/reference/string/to_string/ - // Currently minify and to_string are identical but in the future, they may - // differ. - simdjson::internal::string_builder<> sb; - sb.append(x); - std::string_view answer = sb.str(); - return std::string(answer.data(), answer.size()); +template std::string to_string(T x) { + // in C++, to_string is standard: + // http://www.cplusplus.com/reference/string/to_string/ Currently minify and + // to_string are identical but in the future, they may differ. + simdjson::internal::string_builder<> sb; + sb.append(x); + std::string_view answer = sb.str(); + return std::string(answer.data(), answer.size()); } #if SIMDJSON_EXCEPTIONS -template -std::string to_string(simdjson_result x) { - if (x.error()) { throw simdjson_error(x.error()); } - return to_string(x.value()); +template std::string to_string(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return to_string(x.value()); } #endif /** - * Minifies a JSON element or document, printing the smallest possible valid JSON. + * Minifies a JSON element or document, printing the smallest possible valid + * JSON. * * dom::parser parser; * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); * cout << minify(doc) << endl; // prints [1,2,3] * */ -template -std::string minify(T x) { - return to_string(x); -} +template std::string minify(T x) { return to_string(x); } #if SIMDJSON_EXCEPTIONS -template -std::string minify(simdjson_result x) { - if (x.error()) { throw simdjson_error(x.error()); } - return to_string(x.value()); +template std::string minify(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return to_string(x.value()); } #endif /** - * Prettifies a JSON element or document, printing the valid JSON with indentation. + * Prettifies a JSON element or document, printing the valid JSON with + * indentation. * * dom::parser parser; * element doc = parser.parse(" [ 1 , 2 , 3 ] "_padded); @@ -6877,25 +6936,24 @@ std::string minify(simdjson_result x) { * cout << prettify(doc) << endl; * */ -template -std::string prettify(T x) { - simdjson::internal::string_builder sb; - sb.append(x); - std::string_view answer = sb.str(); - return std::string(answer.data(), answer.size()); +template std::string prettify(T x) { + simdjson::internal::string_builder sb; + sb.append(x); + std::string_view answer = sb.str(); + return std::string(answer.data(), answer.size()); } #if SIMDJSON_EXCEPTIONS -template -std::string prettify(simdjson_result x) { - if (x.error()) { throw simdjson_error(x.error()); } - return to_string(x.value()); +template std::string prettify(simdjson_result x) { + if (x.error()) { + throw simdjson_error(x.error()); + } + return to_string(x.value()); } #endif } // namespace simdjson - #endif /* end file simdjson/dom/serialization.h */ @@ -8927,8 +8985,8 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { #define SIMDJSON_SERIALIZATION_INL_H /* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/serialization.h" */ /* skipped duplicate #include "simdjson/dom/parser.h" */ +/* skipped duplicate #include "simdjson/dom/serialization.h" */ /* skipped duplicate #include "simdjson/internal/tape_type.h" */ /* skipped duplicate #include "simdjson/dom/array-inl.h" */ @@ -8940,7 +8998,9 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { namespace simdjson { namespace dom { inline bool parser::print_json(std::ostream &os) const noexcept { - if (!valid) { return false; } + if (!valid) { + return false; + } simdjson::internal::string_builder<> sb; sb.append(doc.root()); std::string_view answer = sb.str(); @@ -8948,37 +9008,51 @@ inline bool parser::print_json(std::ostream &os) const noexcept { return true; } -inline std::ostream& operator<<(std::ostream& out, simdjson::dom::element value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); +inline std::ostream &operator<<(std::ostream &out, + simdjson::dom::element value) { + simdjson::internal::string_builder<> sb; + sb.append(value); + return (out << sb.str()); } #if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { - if (x.error()) { throw simdjson::simdjson_error(x.error()); } - return (out << x.value()); +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x) { + if (x.error()) { + throw simdjson::simdjson_error(x.error()); + } + return (out << x.value()); } #endif -inline std::ostream& operator<<(std::ostream& out, simdjson::dom::array value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); +inline std::ostream &operator<<(std::ostream &out, simdjson::dom::array value) { + simdjson::internal::string_builder<> sb; + sb.append(value); + return (out << sb.str()); } #if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { - if (x.error()) { throw simdjson::simdjson_error(x.error()); } - return (out << x.value()); +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x) { + if (x.error()) { + throw simdjson::simdjson_error(x.error()); + } + return (out << x.value()); } #endif -inline std::ostream& operator<<(std::ostream& out, simdjson::dom::object value) { - simdjson::internal::string_builder<> sb; - sb.append(value); - return (out << sb.str()); +inline std::ostream &operator<<(std::ostream &out, + simdjson::dom::object value) { + simdjson::internal::string_builder<> sb; + sb.append(value); + return (out << sb.str()); } #if SIMDJSON_EXCEPTIONS -inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { - if (x.error()) { throw simdjson::simdjson_error(x.error()); } - return (out << x.value()); +inline std::ostream & +operator<<(std::ostream &out, + simdjson::simdjson_result x) { + if (x.error()) { + throw simdjson::simdjson_error(x.error()); + } + return (out << x.value()); } #endif @@ -8993,8 +9067,9 @@ namespace { * We expect that most compilers will use 8 bytes for this data structure. **/ struct escape_sequence { - uint8_t length; - const char string[7]; // technically, we only ever need 6 characters, we pad to 8 + uint8_t length; + const char + string[7]; // technically, we only ever need 6 characters, we pad to 8 }; /**@private * This converts a signed integer into a character sequence. @@ -9010,7 +9085,7 @@ static char *fast_itoa(char *output, int64_t value) noexcept { char buffer[20]; uint64_t value_positive; // In general, negating a signed integer is unsafe. - if(value < 0) { + if (value < 0) { *output++ = '-'; // Doing value_positive = -value; while avoiding // undefined behavior warnings. @@ -9029,7 +9104,7 @@ static char *fast_itoa(char *output, int64_t value) noexcept { // A faster approach is possible if we expect large integers: // unroll the loop (work in 100s, 1000s) and use some kind of // memoization. - while(value_positive >= 10) { + while (value_positive >= 10) { *write_pointer-- = char('0' + (value_positive % 10)); value_positive /= 10; } @@ -9055,7 +9130,7 @@ static char *fast_itoa(char *output, uint64_t value) noexcept { // A faster approach is possible if we expect large integers: // unroll the loop (work in 100s, 1000s) and use some kind of // memoization. - while(value >= 10) { + while (value >= 10) { *write_pointer-- = char('0' + (value % 10)); value /= 10; }; @@ -9065,7 +9140,6 @@ static char *fast_itoa(char *output, uint64_t value) noexcept { return output + len; } - } // anonymous namespace namespace internal { @@ -9073,193 +9147,208 @@ namespace internal { * Minifier/formatter code. **/ -template +template simdjson_inline void base_formatter::number(uint64_t x) { char number_buffer[24]; char *newp = fast_itoa(number_buffer, x); - buffer.insert(buffer.end(), number_buffer, newp); + chars(number_buffer, newp); } -template +template simdjson_inline void base_formatter::number(int64_t x) { char number_buffer[24]; char *newp = fast_itoa(number_buffer, x); - buffer.insert(buffer.end(), number_buffer, newp); + chars(number_buffer, newp); } -template +template simdjson_inline void base_formatter::number(double x) { char number_buffer[24]; // Currently, passing the nullptr to the second argument is // safe because our implementation does not check the second // argument. char *newp = internal::to_chars(number_buffer, nullptr, x); - buffer.insert(buffer.end(), number_buffer, newp); + chars(number_buffer, newp); } -template -simdjson_inline void base_formatter::start_array() { one_char('['); } +template +simdjson_inline void base_formatter::start_array() { + one_char('['); +} +template +simdjson_inline void base_formatter::end_array() { + one_char(']'); +} -template -simdjson_inline void base_formatter::end_array() { one_char(']'); } +template +simdjson_inline void base_formatter::start_object() { + one_char('{'); +} -template -simdjson_inline void base_formatter::start_object() { one_char('{'); } +template +simdjson_inline void base_formatter::end_object() { + one_char('}'); +} -template -simdjson_inline void base_formatter::end_object() { one_char('}'); } +template +simdjson_inline void base_formatter::comma() { + one_char(','); +} -template -simdjson_inline void base_formatter::comma() { one_char(','); } - -template +template simdjson_inline void base_formatter::true_atom() { - const char * s = "true"; - buffer.insert(buffer.end(), s, s + 4); + const char *s = "true"; + chars(s, s + 4); } -template +template simdjson_inline void base_formatter::false_atom() { - const char * s = "false"; - buffer.insert(buffer.end(), s, s + 5); + const char *s = "false"; + chars(s, s + 5); } -template +template simdjson_inline void base_formatter::null_atom() { - const char * s = "null"; - buffer.insert(buffer.end(), s, s + 4); + const char *s = "null"; + chars(s, s + 4); } -template -simdjson_inline void base_formatter::one_char(char c) { buffer.push_back(c); } +template +simdjson_inline void base_formatter::one_char(char c) { + buffer.push_back(c); +} -template -simdjson_inline void base_formatter::key(std::string_view unescaped) { +template +simdjson_inline void base_formatter::chars(const char *begin, + const char *end) { + buffer.append(begin, end); +} + +template +simdjson_inline void +base_formatter::key(std::string_view unescaped) { string(unescaped); one_char(':'); } -template -simdjson_inline void base_formatter::string(std::string_view unescaped) { +template +simdjson_inline void +base_formatter::string(std::string_view unescaped) { one_char('\"'); size_t i = 0; - // Fast path for the case where we have no control character, no ", and no backslash. - // This should include most keys. + // Fast path for the case where we have no control character, no ", and no + // backslash. This should include most keys. // - // We would like to use 'bool' but some compilers take offense to bitwise operation - // with bool types. - constexpr static char needs_escaping[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - for(;i + 8 <= unescaped.length(); i += 8) { + // We would like to use 'bool' but some compilers take offense to bitwise + // operation with bool types. + constexpr static char needs_escaping[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + for (; i + 8 <= unescaped.length(); i += 8) { // Poor's man vectorization. This could get much faster if we used SIMD. // - // It is not the case that replacing '|' with '||' would be neutral performance-wise. - if(needs_escaping[uint8_t(unescaped[i])] | needs_escaping[uint8_t(unescaped[i+1])] - | needs_escaping[uint8_t(unescaped[i+2])] | needs_escaping[uint8_t(unescaped[i+3])] - | needs_escaping[uint8_t(unescaped[i+4])] | needs_escaping[uint8_t(unescaped[i+5])] - | needs_escaping[uint8_t(unescaped[i+6])] | needs_escaping[uint8_t(unescaped[i+7])] - ) { break; } + // It is not the case that replacing '|' with '||' would be neutral + // performance-wise. + if (needs_escaping[uint8_t(unescaped[i])] | + needs_escaping[uint8_t(unescaped[i + 1])] | + needs_escaping[uint8_t(unescaped[i + 2])] | + needs_escaping[uint8_t(unescaped[i + 3])] | + needs_escaping[uint8_t(unescaped[i + 4])] | + needs_escaping[uint8_t(unescaped[i + 5])] | + needs_escaping[uint8_t(unescaped[i + 6])] | + needs_escaping[uint8_t(unescaped[i + 7])]) { + break; + } } - for(;i < unescaped.length(); i++) { - if(needs_escaping[uint8_t(unescaped[i])]) { break; } + for (; i < unescaped.length(); i++) { + if (needs_escaping[uint8_t(unescaped[i])]) { + break; + } } - // The following is also possible and omits a 256-byte table, but it is slower: - // for (; (i < unescaped.length()) && (uint8_t(unescaped[i]) > 0x1F) + // The following is also possible and omits a 256-byte table, but it is + // slower: for (; (i < unescaped.length()) && (uint8_t(unescaped[i]) > 0x1F) // && (unescaped[i] != '\"') && (unescaped[i] != '\\'); i++) {} // At least for long strings, the following should be fast. We could // do better by integrating the checks and the insertion. - buffer.insert(buffer.end(), unescaped.data(), unescaped.data() + i); + chars(unescaped.data(), unescaped.data() + i); // We caught a control character if we enter this loop (slow). // Note that we are do not restart from the beginning, but rather we continue // from the point where we encountered something that requires escaping. for (; i < unescaped.length(); i++) { switch (unescaped[i]) { - case '\"': - { - const char * s = "\\\""; - buffer.insert(buffer.end(), s, s + 2); - } - break; - case '\\': - { - const char * s = "\\\\"; - buffer.insert(buffer.end(), s, s + 2); - } - break; + case '\"': { + const char *s = "\\\""; + chars(s, s + 2); + } break; + case '\\': { + const char *s = "\\\\"; + chars(s, s + 2); + } break; default: if (uint8_t(unescaped[i]) <= 0x1F) { // If packed, this uses 8 * 32 bytes. // Note that we expect most compilers to embed this code in the data // section. constexpr static escape_sequence escaped[32] = { - {6, "\\u0000"}, {6, "\\u0001"}, {6, "\\u0002"}, {6, "\\u0003"}, - {6, "\\u0004"}, {6, "\\u0005"}, {6, "\\u0006"}, {6, "\\u0007"}, - {2, "\\b"}, {2, "\\t"}, {2, "\\n"}, {6, "\\u000b"}, - {2, "\\f"}, {2, "\\r"}, {6, "\\u000e"}, {6, "\\u000f"}, - {6, "\\u0010"}, {6, "\\u0011"}, {6, "\\u0012"}, {6, "\\u0013"}, - {6, "\\u0014"}, {6, "\\u0015"}, {6, "\\u0016"}, {6, "\\u0017"}, - {6, "\\u0018"}, {6, "\\u0019"}, {6, "\\u001a"}, {6, "\\u001b"}, - {6, "\\u001c"}, {6, "\\u001d"}, {6, "\\u001e"}, {6, "\\u001f"}}; + {6, "\\u0000"}, {6, "\\u0001"}, {6, "\\u0002"}, {6, "\\u0003"}, + {6, "\\u0004"}, {6, "\\u0005"}, {6, "\\u0006"}, {6, "\\u0007"}, + {2, "\\b"}, {2, "\\t"}, {2, "\\n"}, {6, "\\u000b"}, + {2, "\\f"}, {2, "\\r"}, {6, "\\u000e"}, {6, "\\u000f"}, + {6, "\\u0010"}, {6, "\\u0011"}, {6, "\\u0012"}, {6, "\\u0013"}, + {6, "\\u0014"}, {6, "\\u0015"}, {6, "\\u0016"}, {6, "\\u0017"}, + {6, "\\u0018"}, {6, "\\u0019"}, {6, "\\u001a"}, {6, "\\u001b"}, + {6, "\\u001c"}, {6, "\\u001d"}, {6, "\\u001e"}, {6, "\\u001f"}}; auto u = escaped[uint8_t(unescaped[i])]; - buffer.insert(buffer.end(), u.string, u.string + u.length); + chars(u.string, u.string + u.length); } else { one_char(unescaped[i]); } } // switch - } // for + } // for one_char('\"'); } - -template -inline void base_formatter::clear() { +template inline void base_formatter::clear() { buffer.clear(); } -template +template simdjson_inline std::string_view base_formatter::str() const { - return std::string_view(buffer.data(), buffer.size()); + return buffer.str(); } -simdjson_inline void mini_formatter::print_newline() { - return; -} +simdjson_inline void mini_formatter::print_newline() { return; } simdjson_inline void mini_formatter::print_indents(size_t depth) { - (void)depth; - return; + (void)depth; + return; } -simdjson_inline void mini_formatter::print_space() { - return; -} +simdjson_inline void mini_formatter::print_space() { return; } -simdjson_inline void pretty_formatter::print_newline() { - one_char('\n'); -} +simdjson_inline void pretty_formatter::print_newline() { one_char('\n'); } simdjson_inline void pretty_formatter::print_indents(size_t depth) { - if(this->indent_step <= 0) { - return; - } - for(size_t i = 0; i < this->indent_step * depth; i++) { - one_char(' '); - } + if (this->indent_step <= 0) { + return; + } + for (size_t i = 0; i < this->indent_step * depth; i++) { + one_char(' '); + } } -simdjson_inline void pretty_formatter::print_space() { - one_char(' '); -} +simdjson_inline void pretty_formatter::print_space() { one_char(' '); } /*** * String building code. @@ -9438,7 +9527,8 @@ inline void string_builder::append(simdjson::dom::array value) { } template -simdjson_inline void string_builder::append(simdjson::dom::key_value_pair kv) { +simdjson_inline void +string_builder::append(simdjson::dom::key_value_pair kv) { format.key(kv.key); append(kv.value); } @@ -9453,7 +9543,6 @@ simdjson_inline std::string_view string_builder::str() const { return format.str(); } - } // namespace internal } // namespace simdjson @@ -32471,35 +32560,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -34808,6 +34897,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace arm64 { @@ -35014,13 +35104,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -35050,6 +35133,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -35159,13 +35244,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -35532,7 +35643,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -35565,7 +35676,7 @@ struct simdjson_result : public arm64::implemen simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -36760,6 +36871,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -36773,6 +36885,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -36816,6 +36929,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -36827,6 +36945,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -37612,7 +37731,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -40221,10 +40340,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -41969,6 +42097,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -42013,6 +42148,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace arm64 } // namespace simdjson @@ -44814,6 +44977,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace arm64 +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return arm64::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -45222,35 +45390,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -47559,6 +47727,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace fallback { @@ -47765,13 +47934,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -47801,6 +47963,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -47910,13 +48074,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -48283,7 +48473,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -48316,7 +48506,7 @@ struct simdjson_result : public fallback::im simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -49511,6 +49701,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -49524,6 +49715,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -49567,6 +49759,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -49578,6 +49775,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -50363,7 +50561,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -52972,10 +53170,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -54720,6 +54927,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -54764,6 +54978,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace fallback } // namespace simdjson @@ -57565,6 +57807,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace fallback +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return fallback::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -58472,35 +58719,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -60809,6 +61056,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace haswell { @@ -61015,13 +61263,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -61051,6 +61292,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -61160,13 +61403,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -61533,7 +61802,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -61566,7 +61835,7 @@ struct simdjson_result : public haswell::impl simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -62761,6 +63030,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -62774,6 +63044,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -62817,6 +63088,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -62828,6 +63104,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -63613,7 +63890,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -66222,10 +66499,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -67970,6 +68256,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -68014,6 +68307,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace haswell } // namespace simdjson @@ -70815,6 +71136,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace haswell +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return haswell::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -71722,35 +72048,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -74059,6 +74385,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace icelake { @@ -74265,13 +74592,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -74301,6 +74621,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -74410,13 +74732,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -74783,7 +75131,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -74816,7 +75164,7 @@ struct simdjson_result : public icelake::impl simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -76011,6 +76359,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -76024,6 +76373,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -76067,6 +76417,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -76078,6 +76433,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -76863,7 +77219,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -79472,10 +79828,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -81220,6 +81585,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -81264,6 +81636,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace icelake } // namespace simdjson @@ -84065,6 +84465,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace icelake +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return icelake::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -85087,35 +85492,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -87424,6 +87829,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace ppc64 { @@ -87630,13 +88036,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -87666,6 +88065,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -87775,13 +88176,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -88148,7 +88575,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -88181,7 +88608,7 @@ struct simdjson_result : public ppc64::implemen simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -89376,6 +89803,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -89389,6 +89817,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -89432,6 +89861,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -89443,6 +89877,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -90228,7 +90663,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -92837,10 +93272,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -94585,6 +95029,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -94629,6 +95080,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace ppc64 } // namespace simdjson @@ -97430,6 +97909,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace ppc64 +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return ppc64::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -98768,35 +99252,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -101105,6 +101589,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace westmere { @@ -101311,13 +101796,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -101347,6 +101825,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -101456,13 +101936,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -101829,7 +102335,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -101862,7 +102368,7 @@ struct simdjson_result : public westmere::im simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -103057,6 +103563,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -103070,6 +103577,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -103113,6 +103621,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -103124,6 +103637,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -103909,7 +104423,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -106518,10 +107032,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -108266,6 +108789,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -108310,6 +108840,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace westmere } // namespace simdjson @@ -111111,6 +111669,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace westmere +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return westmere::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -111926,35 +112489,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -114263,6 +114826,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace lsx { @@ -114469,13 +115033,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -114505,6 +115062,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -114614,13 +115173,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -114987,7 +115572,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -115020,7 +115605,7 @@ struct simdjson_result : public lsx::implementati simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -116215,6 +116800,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -116228,6 +116814,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -116271,6 +116858,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -116282,6 +116874,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -117067,7 +117660,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -119676,10 +120269,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -121424,6 +122026,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -121468,6 +122077,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace lsx } // namespace simdjson @@ -124269,6 +124906,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace lsx +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return lsx::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -125097,35 +125739,35 @@ inline constexpr struct deserialize_tag { // Customization Point for array template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(array_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for object template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(object_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for value template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } // Customization Point for document reference template requires custom_deserializable - [[nodiscard]] constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { + simdjson_warn_unused constexpr /* error_code */ auto operator()(document_reference_type &object, T& output) const noexcept(nothrow_custom_deserializable) { return tag_invoke(*this, object, output); } @@ -127434,6 +128076,7 @@ public: /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #include +#include namespace simdjson { namespace lasx { @@ -127640,13 +128283,6 @@ public: * Setting batch_size to excessively large or excessively small values may impact negatively the * performance. * - * ### REQUIRED: Buffer Padding - * - * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what - * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you - * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the - * SIMDJSON_PADDING bytes to avoid runtime warnings. - * * ### Threads * * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the @@ -127676,6 +128312,8 @@ public: */ inline simdjson_result iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; @@ -127785,13 +128423,39 @@ public: bool string_buffer_overflow(const uint8_t *string_buf_loc) const noexcept; #endif + /** + * Get a unique parser instance corresponding to the current thread. + * This instance can be safely used within the current thread, but it should + * not be passed to other threads. + * + * A parser should only be used for one document at a time. + * + * Our simdjson::from functions use this parser instance. + * + * You can free the related parser by calling release_parser(). + */ + static simdjson_inline simdjson_warn_unused ondemand::parser& get_parser(); + /** + * Release the parser instance initialized by get_parser() and all the + * associated resources (memory). Returns true if a parser instance + * was released. + */ + static simdjson_inline bool release_parser(); + private: + friend bool release_parser(); + friend ondemand::parser& get_parser(); + /** Get the thread-local parser instance, allocates it if needed */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_parser_instance(); + /** Get the thread-local parser instance, it might be null */ + static simdjson_inline simdjson_warn_unused std::unique_ptr& get_threadlocal_parser_if_exists(); /** @private [for benchmarking access] The implementation to use */ std::unique_ptr implementation{}; size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; std::unique_ptr string_buf{}; + #if SIMDJSON_DEVELOPMENT_CHECKS std::unique_ptr start_positions{}; #endif @@ -128158,7 +128822,7 @@ public: /** * Check if the array is at the end. */ - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; private: value_iterator iter{}; @@ -128191,7 +128855,7 @@ struct simdjson_result : public lasx::implementa simdjson_inline bool operator!=(const simdjson_result &) const noexcept; simdjson_inline simdjson_result &operator++() noexcept; - [[nodiscard]] simdjson_inline bool at_end() const noexcept; + simdjson_warn_unused simdjson_inline bool at_end() const noexcept; }; } // namespace simdjson @@ -129386,6 +130050,7 @@ public: * Default constructor. */ simdjson_inline iterator() noexcept; + simdjson_inline iterator(const iterator &other) noexcept = default; /** * Get the current document (or error). */ @@ -129399,6 +130064,7 @@ public: * @param other the end iterator to compare to. */ simdjson_inline bool operator!=(const iterator &other) const noexcept; + simdjson_inline bool operator==(const iterator &other) const noexcept; /** * @private * @@ -129442,6 +130108,11 @@ public: */ inline error_code error() const noexcept; + /** + * Returns whether the iterator is at the end. + */ + inline bool at_end() const noexcept; + private: simdjson_inline iterator(document_stream *s, bool finished) noexcept; /** The document_stream we're iterating through. */ @@ -129453,6 +130124,7 @@ public: friend class document_stream; friend class json_iterator; }; + using iterator = document_stream::iterator; /** * Start iterating the documents in the stream. @@ -130238,7 +130910,7 @@ inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result #include #if SIMDJSON_STATIC_REFLECTION -#include +#include // #include // for std::define_static_string - header not available yet #endif @@ -132847,10 +133519,19 @@ simdjson_inline document_stream::iterator& document_stream::iterator::operator++ return *this; } +simdjson_inline bool document_stream::iterator::at_end() const noexcept { + return finished; +} + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator &other) const noexcept { return finished != other.finished; } +simdjson_inline bool document_stream::iterator::operator==(const document_stream::iterator &other) const noexcept { + return finished == other.finished; +} + simdjson_inline document_stream::iterator document_stream::begin() noexcept { start(); // If there are no documents, we're finished. @@ -134595,6 +135276,13 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, if(allow_comma_separated && batch_size < len) { batch_size = len; } return document_stream(*this, buf, len, batch_size, allow_comma_separated); } + +inline simdjson_result parser::iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if(allow_comma_separated && batch_size < json.length()) { batch_size = json.length(); } + return iterate_many(json.data(), json.length(), batch_size, allow_comma_separated); +} + inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); } @@ -134639,6 +135327,34 @@ simdjson_inline simdjson_warn_unused simdjson_result parser::u return result; } +simdjson_inline simdjson_warn_unused ondemand::parser& parser::get_parser() { + return *parser::get_parser_instance(); +} + +simdjson_inline bool release_parser() { + auto &parser_instance = parser::get_threadlocal_parser_if_exists(); + if (parser_instance) { + parser_instance.reset(); + return true; + } + return false; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_parser_instance() { + std::unique_ptr& parser_instance = get_threadlocal_parser_if_exists(); + if (!parser_instance) { + parser_instance.reset(new ondemand::parser()); + } + return parser_instance; +} + +simdjson_inline simdjson_warn_unused std::unique_ptr& parser::get_threadlocal_parser_if_exists() { + // @the-moisrex points out that this could be implemented with std::optional (C++17). + thread_local std::unique_ptr parser_instance = nullptr; + return parser_instance; +} + + } // namespace ondemand } // namespace lasx } // namespace simdjson @@ -137440,6 +138156,11 @@ string_builder& operator<<(string_builder& b, const Z& z) { } } // namespace builder } // namespace lasx +// Alias the function template to 'to' in the global namespace +template +simdjson_result to_json(const Z &z, size_t initial_capacity = 1024) { + return lasx::builder::to_json_string(z, initial_capacity); +} } // namespace simdjson #endif // SIMDJSON_STATIC_REFLECTION @@ -137520,76 +138241,11 @@ namespace simdjson { /* skipped duplicate #include "simdjson/ondemand.h" */ #include -#ifdef __cpp_lib_ranges -#include -#endif namespace simdjson { -struct [[nodiscard]] auto_iterator_end {}; - -/** - * A Wrapper for simdjson_result in order to make it - * compatible with ranges (to satisfy std::ranges::input_range). - */ -struct [[nodiscard]] auto_iterator { - using iterator_category = std::forward_iterator_tag; - using type = simdjson_result; - using value_type = simdjson_result; // type::value_type - using reference = value_type &; - using const_reference = const value_type &; - using difference_type = std::ptrdiff_t; - - struct auto_iterator_storage { - type m_iter{}; - mutable value_type m_value{}; - }; - -private: - auto_iterator_storage *m_storage = nullptr; - -public: - constexpr auto_iterator() noexcept = default; - explicit auto_iterator(auto_iterator_storage &storage) noexcept - : m_storage{&storage} {}; - auto_iterator(auto_iterator const &) = default; - auto_iterator(auto_iterator &&) = default; - auto_iterator &operator=(auto_iterator const &) = default; - auto_iterator &operator=(auto_iterator &&) noexcept = default; - ~auto_iterator() = default; - - reference operator*() const noexcept { return m_storage->m_value; } - reference operator*() noexcept { return m_storage->m_value; } - - auto_iterator &operator++() noexcept { - ++m_storage->m_iter; - m_storage->m_value = - m_storage->m_iter.at_end() || m_storage->m_iter.error() != SUCCESS - ? value_type{} - : *m_storage->m_iter; - return *this; - } - auto_iterator operator++(int) noexcept { - auto_iterator const tmp = *this; - operator++(); - return tmp; - } - - [[nodiscard]] bool operator==(auto_iterator const &other) const noexcept { - return m_storage == other.m_storage && - m_storage->m_iter == other.m_storage->m_iter; - } - - [[nodiscard]] bool operator==(auto_iterator_end) const noexcept { - return m_storage != nullptr && m_storage->m_iter.at_end(); - } -}; - -template -struct [[nodiscard]] auto_parser -#if __cpp_lib_ranges - : std::ranges::view_interface> -#endif +template +struct auto_parser { using value_type = simdjson_result; using size_type = size_t; @@ -137598,17 +138254,12 @@ struct [[nodiscard]] auto_parser using const_pointer = const value_type *; using reference = value_type &; using const_reference = const value_type &; - using iterator = auto_iterator; - using const_iterator = auto_iterator; // auto_iterator is already const private: ParserType m_parser; ondemand::document m_doc; error_code m_error{SUCCESS}; - // Caching the iterator here: - iterator::auto_iterator_storage iter_storage{}; - template static constexpr bool is_nothrow_gettable = requires(ondemand::document doc) { { doc.get() } noexcept; @@ -137620,57 +138271,29 @@ public: requires(!std::is_pointer_v) : m_parser{std::move(parser)}, m_doc{std::move(doc)} {} -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Weffc++" -#endif - explicit auto_parser(ParserType &&parser, padded_string_view const str) noexcept requires(!std::is_pointer_v) : m_parser{std::move(parser)}, m_doc{}, m_error{SUCCESS} { - // Initialize m_doc after m_parser to avoid potential issues - auto doc_result = m_parser.iterate(str); - m_error = doc_result.error(); - if (m_error == SUCCESS) { - m_doc = std::move(doc_result.value_unsafe()); - } + m_error = m_parser.iterate(str).get(m_doc); } -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - - explicit auto_parser(padded_string_view const str) noexcept - requires(!std::is_pointer_v) - : auto_parser{ParserType{}, str} {} - // pointer constructors: explicit auto_parser(std::remove_pointer_t &parser, ondemand::document &&doc) noexcept requires(std::is_pointer_v) : m_parser{&parser}, m_doc{std::move(doc)} {} -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Weffc++" -#endif - explicit auto_parser(std::remove_pointer_t &parser, padded_string_view const str) noexcept requires(std::is_pointer_v) : m_parser{&parser}, m_doc{}, m_error{SUCCESS} { - // Initialize m_doc after m_parser to avoid potential issues - auto doc_result = parser.iterate(str); - m_error = doc_result.error(); - if (m_error == SUCCESS) { - m_doc = std::move(doc_result.value_unsafe()); - } + m_error = m_parser->iterate(str).get(m_doc); } -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif + explicit auto_parser(padded_string_view const str) noexcept + requires(std::is_pointer_v) + : auto_parser{ondemand::parser::get_parser(), str} {} explicit auto_parser(ParserType parser, ondemand::document &&doc) noexcept requires(std::is_pointer_v) @@ -137683,7 +138306,7 @@ public: ~auto_parser() = default; /// Get the parser - [[nodiscard]] std::remove_pointer_t &parser() noexcept { + simdjson_warn_unused std::remove_pointer_t &parser() noexcept { if constexpr (std::is_pointer_v) { return *m_parser; } else { @@ -137692,7 +138315,7 @@ public: } template - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result result() noexcept(is_nothrow_gettable) { if (m_error != SUCCESS) { return m_error; @@ -137701,29 +138324,29 @@ public: return m_doc.get(); } - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result array() noexcept { return result(); } - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result object() noexcept { return result(); } - [[nodiscard]] simdjson_inline simdjson_result + simdjson_warn_unused simdjson_inline simdjson_result number() noexcept { return result(); } template - [[nodiscard]] simdjson_inline explicit(false) + simdjson_warn_unused simdjson_inline explicit(false) operator simdjson_result() noexcept(is_nothrow_gettable) { return result(); } template - [[nodiscard]] simdjson_inline explicit(false) operator T() noexcept(false) { + simdjson_warn_unused simdjson_inline explicit(false) operator T() noexcept(false) { if (m_error != SUCCESS) { throw simdjson_error(m_error); } @@ -137736,81 +138359,37 @@ public: // We also cannot have "operator T&" without manual memory management either. template - [[nodiscard]] simdjson_inline std::optional + simdjson_warn_unused simdjson_inline std::optional optional() noexcept(is_nothrow_gettable) { if (m_error != SUCCESS) { return std::nullopt; } + T value; // For std::optional - auto res = m_doc.get(); - if (res.error()) [[unlikely]] { + if (m_doc.get().get(value)) [[unlikely]] { return std::nullopt; } - return {res.value()}; + return {std::move(value)}; } - - simdjson_inline auto_iterator begin() noexcept { - if (m_error != SUCCESS) { - // Create an iterator with the error - iter_storage.m_iter = iterator::type(m_error); - iter_storage.m_value = value_type{}; - return auto_iterator{iter_storage}; - } - if (iter_storage.m_iter.error() != SUCCESS && - !iter_storage.m_iter.at_end()) { - // Try to get the document as an array - auto array_result = m_doc.get_array(); - if (array_result.error() == SUCCESS) { - iter_storage = {.m_iter = iterator::type{array_result.value_unsafe().begin()}, - .m_value = iterator::value_type{ - iter_storage.m_iter.at_end() || - iter_storage.m_iter.error() != SUCCESS - ? value_type{} - : *iter_storage.m_iter}}; - } else { - // If it's not an array, create an error iterator - iter_storage.m_iter = iterator::type(array_result.error()); - iter_storage.m_value = value_type{}; - } - } - return auto_iterator{iter_storage}; - } - simdjson_inline auto_iterator_end end() noexcept { return {}; } }; -#ifdef __cpp_lib_ranges -// For C++20, we implement our own pipe operator since range_adaptor_closure is C++23 -static constexpr struct [[nodiscard]] no_errors_adaptor { - - [[nodiscard]] bool - operator()(simdjson_result const &val) const noexcept { - return val.error() == SUCCESS; - } - - template - auto operator()(Range &&rng) const noexcept { - return std::forward(rng) | std::views::filter(*this); - } -} no_errors; template -struct [[nodiscard]] to_adaptor { +struct to_adaptor { /// Convert to T - [[nodiscard]] T - operator()(simdjson_result &val) const noexcept { + T operator()(simdjson_result &val) const noexcept { return val.get(); } - - /// Make it an adaptor - template - auto operator()(Range &&rng) const noexcept { - return std::forward(rng) | no_errors | std::views::transform(*this); - } - /** - * Parse input string into any object if possible. + * Parse input string into any object if possible. This function call + * will return an auto_parser that can be used to extract the desired + * object from the input string. The ondemand::parser instance is created + * internally. + * + * This function uses the simdjson::ondemand::parser::get_parser() instance. + * A parser should only be used for one document at a time. */ auto operator()(padded_string_view const str) const noexcept { return auto_parser{str}; @@ -137818,6 +138397,9 @@ struct [[nodiscard]] to_adaptor { /** * Parse the input using the specified parser into any object if possible. + * This function call will return an auto_parser that can be used to extract + * the desired object from the input string. The provided ondemand::parser instance + * handles memory allocations and indexing, and it can be reused from call to call. */ auto operator()(ondemand::parser &parser, padded_string_view const str) const noexcept { @@ -137827,21 +138409,32 @@ struct [[nodiscard]] to_adaptor { template static constexpr to_adaptor to{}; +/** + * The `from` instance is a utility adaptor for parsing JSON strings into objects. + * It provides a convenient way to convert JSON data into C++ objects using the `auto_parser`. + * + * Example usage: + * + * ```cpp + * std::map obj = + * simdjson::from(R"({"key": "value"})"_padded); + * ``` + * + * This will parse the JSON string and return an object representation. By default, we + * use the simdjson::ondemand::parser::get_parser() instance. A parser instance should + * be used for just one document at a time. + * + * You can also pass you own parser instance: + * ```cpp + * simdjson::ondemand::parser parser; + * std::map obj = + * simdjson::from(parser, R"({"key": "value"})"_padded); + * ``` + * The parser instance can be reused. + * + */ static constexpr to_adaptor<> from{}; -// For C++20 ranges without range_adaptor_closure, we need to define pipe operators -template -inline auto operator|(Range&& range, const no_errors_adaptor& adaptor) { - return adaptor(std::forward(range)); -} - -template -inline auto operator|(Range&& range, const to_adaptor& adaptor) { - return adaptor(std::forward(range)); -} - -#endif // __cpp_lib_ranges - } // namespace simdjson #endif // __cpp_concepts diff --git a/singleheader/singleheader.zip b/singleheader/singleheader.zip index 14b0ab204..1cb541933 100644 Binary files a/singleheader/singleheader.zip and b/singleheader/singleheader.zip differ diff --git a/tests/ondemand/ondemand_convert_tests.cpp b/tests/ondemand/ondemand_convert_tests.cpp index 15eb87455..ec5c89584 100644 --- a/tests/ondemand/ondemand_convert_tests.cpp +++ b/tests/ondemand/ondemand_convert_tests.cpp @@ -61,27 +61,6 @@ struct Car { static_assert(simdjson::custom_deserializable>, "It should be deserializable"); -static_assert(std::input_or_output_iterator, - "Must be a valid input iterator"); -static_assert(std::semiregular, - "Should be kinda regular"); -// static_assert(std::ranges::__access::__member_end>, -// "Must be a valid input iterator"); -// static_assert(std::ranges::views::__adaptor::__is_range_adaptor_closure< -// simdjson::auto_parser<>>, -// "Parser need to be range adaptor closure."); -// static_assert(std::ranges::views::__adaptor::__adaptor_invocable< -// decltype(simdjson::to()), -// simdjson::auto_parser<>>, -// "I don't even know!"); -static_assert(std::ranges::range>, - "Parser need to be a range."); -static_assert(std::ranges::forward_range>, - "Parser need to be an input range."); -static_assert( - requires(simdjson::auto_parser<> &parser) { - { parser.begin() } -> std::input_or_output_iterator; - }, "Must be valid iterator."); simdjson::padded_string json_car = R"( { @@ -165,23 +144,6 @@ bool to_array() { TEST_SUCCEED(); } -bool to_array_shortcut() { - TEST_START(); - simdjson::ondemand::parser parser; - for (auto val : simdjson::from(parser, json_cars)) { - Car car{}; - if (auto const error = val.get(car)) { - std::cerr << simdjson::error_message(error) << std::endl; - return false; - } - if (car.year < 1998) { - std::cerr << car.make << " " << car.model << " " << car.year << std::endl; - return false; - } - } - TEST_SUCCEED(); -} - bool to_bad_array() { TEST_START(); auto parser = simdjson::from(json_car); @@ -223,43 +185,17 @@ bool to_bad_array() { TEST_SUCCEED(); } -bool test_basic_adaptor() { - TEST_START(); - for (Car car : simdjson::from(json_cars) | simdjson::as()) { - if (car.year < 1998) { - return false; - } - } - TEST_SUCCEED(); -} - -bool test_basic_adaptor_with_parser() { +#if SIMDJSON_SUPPORTS_RANGES +// currently not active (SIMDJSON_SUPPORTS_RANGES) +bool to_array_shortcut() { TEST_START(); simdjson::ondemand::parser parser; - for (Car car : simdjson::from(parser, json_cars) | simdjson::as()) { - if (car.year < 1998) { + for (auto val : simdjson::from(parser, json_cars)) { + Car car{}; + if (auto const error = val.get(car)) { + std::cerr << simdjson::error_message(error) << std::endl; return false; } - } - TEST_SUCCEED(); -} - -bool test_no_errors() { - TEST_START(); - auto cars = simdjson::from(json_cars) | simdjson::no_errors; - for (auto val : cars) { - Car car = val.get(); - if (car.year < 1998) { - return false; - } - } - TEST_SUCCEED(); -} - -bool to_clean_array() { - TEST_START(); - for (auto val : simdjson::from(json_cars) | simdjson::no_errors) { - Car car = val.get(); if (car.year < 1998) { std::cerr << car.make << " " << car.model << " " << car.year << std::endl; return false; @@ -267,7 +203,70 @@ bool to_clean_array() { } TEST_SUCCEED(); } +// currently not active (SIMDJSON_SUPPORTS_RANGES) +bool test_basic_adaptor() { + TEST_START(); + int64_t sum_year = 0; + for (Car car : simdjson::from(json_cars) | simdjson::as()) { + printf("Car: %s %s %d\n", car.make.c_str(), car.model.c_str(), car.year); + if (car.year < 1998) { + return false; + } + sum_year += car.year; + } + ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999); + TEST_SUCCEED(); +} +// currently not active (SIMDJSON_SUPPORTS_RANGES) +bool test_basic_adaptor_with_parser() { + TEST_START(); + simdjson::ondemand::parser parser; + int64_t sum_year = 0; + for (Car car : simdjson::from(parser, json_cars) | simdjson::as()) { + if (car.year < 1998) { + return false; + } + sum_year += car.year; + } + ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999); + TEST_SUCCEED(); +} + +// currently not active (SIMDJSON_SUPPORTS_RANGES) +bool test_no_errors() { + TEST_START(); + int64_t sum_year = 0; + auto cars = simdjson::from(json_cars) | simdjson::no_errors; + for (auto val : cars) { + Car car = val.get(); + if (car.year < 1998) { + return false; + } + printf("-- year: %d\n", car.year); + sum_year += car.year; + } + ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999); + TEST_SUCCEED(); +} + +// currently not active (SIMDJSON_SUPPORTS_RANGES) +bool to_clean_array() { + TEST_START(); + int64_t sum_year = 0; + for (auto val : simdjson::from(json_cars) | simdjson::no_errors) { + Car car = val.get(); + if (car.year < 1998) { + std::cerr << car.make << " " << car.model << " " << car.year << std::endl; + return false; + } + sum_year += car.year; + } + ASSERT_EQUAL(sum_year, 2018 + 2012 + 1999); + TEST_SUCCEED(); +} + +// currently not active (SIMDJSON_SUPPORTS_RANGES) bool test_to_adaptor_basic() { TEST_START(); // Test 1: Basic usage of to with a value reference @@ -287,6 +286,8 @@ bool test_to_adaptor_basic() { TEST_SUCCEED(); } +#endif // SIMDJSON_SUPPORTS_RANGES + bool test_to_adaptor_with_single_value() { TEST_START(); // Test 2: Using to to convert individual values @@ -338,12 +339,15 @@ bool test_to_vs_from_equivalence() { bool run() { return #if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION - test_basic_adaptor() && broken() && simple() && simple_optional() && with_parser() && to_array() && - to_array_shortcut() && to_bad_array() && test_no_errors() && - to_clean_array() && test_to_adaptor_basic() && + broken() && simple() && simple_optional() && with_parser() && to_array() && + to_bad_array() && test_to_adaptor_with_single_value() && test_to_vs_from_equivalence() && - test_basic_adaptor_with_parser() && example_with_parser() && -#endif // SIMDJSON_EXCEPTIONS + example_with_parser() && +#if SIMDJSON_SUPPORTS_RANGES + test_basic_adaptor_with_parser() && test_no_errors() && test_basic_adaptor() + && test_to_adaptor_basic() && to_clean_array() && to_array_shortcut() && +#endif // SIMDJSON_SUPPORTS_RANGES +#endif // SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_DESERIALIZATION true; }