From 6db330c1a238994b33a01a4b393e50abe469b873 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 25 Oct 2023 19:34:52 -0400 Subject: [PATCH] Automating string reallocation when it is possible (#2076) * Automating string reallocation * Typo --- doc/basics.md | 4 +- .../simdjson/generic/ondemand/parser-inl.h | 7 ++ include/simdjson/generic/ondemand/parser.h | 2 + tests/ondemand/ondemand_parse_api_tests.cpp | 12 ++- tests/ondemand/ondemand_readme_examples.cpp | 73 +++++++++++++++++++ 5 files changed, 93 insertions(+), 5 deletions(-) diff --git a/doc/basics.md b/doc/basics.md index b2f60e288..6194b77c6 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -179,8 +179,8 @@ strcpy(json, "[1]"); ondemand::document doc = parser.iterate(json, strlen(json), sizeof(json)); ``` -The simdjson library will also accept `std::string` instances, as long as the `capacity()` of -the string exceeds the `size()` by at least `SIMDJSON_PADDING`. You can increase the `capacity()` with the `reserve()` function of your strings. +The simdjson library will also accept `std::string` instances. If the provided +reference is non-const, it will allocate padding as needed. You can copy your data directly on a `simdjson::padded_string` as follows: diff --git a/include/simdjson/generic/ondemand/parser-inl.h b/include/simdjson/generic/ondemand/parser-inl.h index 8d68b060d..2913b1d9b 100644 --- a/include/simdjson/generic/ondemand/parser-inl.h +++ b/include/simdjson/generic/ondemand/parser-inl.h @@ -68,6 +68,13 @@ simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(s return iterate(padded_string_view(json, allocated)); } +simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(std::string &json) & noexcept { + if(json.capacity() - json.size() < SIMDJSON_PADDING) { + json.reserve(json.size() + SIMDJSON_PADDING); + } + return iterate(padded_string_view(json)); +} + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const std::string &json) & noexcept { return iterate(padded_string_view(json)); } diff --git a/include/simdjson/generic/ondemand/parser.h b/include/simdjson/generic/ondemand/parser.h index fca6005bc..31484b68e 100644 --- a/include/simdjson/generic/ondemand/parser.h +++ b/include/simdjson/generic/ondemand/parser.h @@ -107,6 +107,8 @@ public: /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const std::string &json) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(std::string &json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const simdjson_result &json) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const simdjson_result &json) & noexcept; diff --git a/tests/ondemand/ondemand_parse_api_tests.cpp b/tests/ondemand/ondemand_parse_api_tests.cpp index 09b4cd900..2e06adc8e 100644 --- a/tests/ondemand/ondemand_parse_api_tests.cpp +++ b/tests/ondemand/ondemand_parse_api_tests.cpp @@ -144,14 +144,20 @@ namespace parse_api_tests { } { - std::string json = "12"; + std::string json = "12345642314123421321321321321321312321321321321312"; json.shrink_to_fit(); cout << "- string, 0 padding" << endl; - ASSERT_ERROR( parser.iterate(json), INSUFFICIENT_PADDING ); + ASSERT_SUCCESS( parser.iterate(json) ); + } + + { + std::string json = "12345642314123421321321321321321312321321321321312"; + json.shrink_to_fit(); + cout << "- string, 0 padding" << endl; + ASSERT_ERROR( parser.iterate((const std::string&)json), INSUFFICIENT_PADDING ); // It's actually kind of hard to allocate "just enough" capacity, since the string tends // to grow more than you tell it to. } - TEST_SUCCEED(); } diff --git a/tests/ondemand/ondemand_readme_examples.cpp b/tests/ondemand/ondemand_readme_examples.cpp index 7deebfdd9..03eee600d 100644 --- a/tests/ondemand/ondemand_readme_examples.cpp +++ b/tests/ondemand/ondemand_readme_examples.cpp @@ -38,6 +38,7 @@ bool to_string_example_no_except() { #if SIMDJSON_EXCEPTIONS + bool to_string_example() { TEST_START(); auto json = R"({ @@ -51,6 +52,7 @@ bool to_string_example() { ASSERT_EQUAL(name, "Daniel"); TEST_SUCCEED(); } + bool gen_raw1() { TEST_START(); simdjson::ondemand::parser parser; @@ -171,6 +173,76 @@ bool examplecrt() { TEST_SUCCEED(); } + +bool examplecrt_realloc() { + TEST_START(); + std::string unpadded_input_json = R"([ + { "monitor": [ + { "id": "monitor", "type": "toggle", "label": "monitor" }, + { "id": "profile", "type": "selector", "label": "collection" }, + { "id": "overlay", "type": "selector", "label": "overlay" }, + { "id": "zoom", "type": "toggleSlider", "label": "zoom" } + ] }, + + { "crt": [ + { "id": "system", "type": "multi", "label": "system", "choices": "PAL, NTSC" }, + { "type": "spacer" }, + { "id": "brightness", "type": "slider", "icon": "brightness" }, + { "id": "contrast", "type": "slider", "icon": "contrast" }, + { "id": "saturation", "type": "slider", "icon": "saturation" }, + { "type": "spacer" }, + { "id": "overscan", "type": "toggleSlider", "label": "overscan" }, + { "type": "spacer" }, + { "id": "emulation", "type": "toggle", "label": "CRT emulation" }, + { "type": "spacer" }, + { "id": "curve", "type": "toggleSlider", "label": "curve" }, + { "id": "bleed", "type": "toggleSlider", "label": "bleed" }, + { "id": "vignette", "type": "toggleSlider", "label": "vignette" }, + { "id": "scanlines", "type": "toggleSlider", "label": "scanlines" }, + { "id": "gridlines", "type": "toggleSlider", "label": "gridlines" }, + { "id": "glow", "type": "toggleSlider", "label": "glow" }, + { "id": "flicker", "type": "toggleSlider", "label": "flicker" }, + { "id": "noise", "type": "toggleSlider", "label": "noise" }, + {} + ] } +])"; + unpadded_input_json.shrink_to_fit(); + auto parser = ondemand::parser{}; + auto doc = parser.iterate(unpadded_input_json); + auto root_array = doc.get_array(); + // the root should be an object, not an array, but that's the JSON we are + // given. + for (ondemand::object node : root_array) { + // We know that we are going to have just one element in the object. + for (auto field : node) { + std::cout << "\n\ntop level:" << field.key() << std::endl; + // You can get a proper std::string_view for the key with: + // std::string_view key = field.unescaped_key(); + // and second for-range loop to get child-elements here + for (ondemand::object inner_object : field.value()) { + auto i = inner_object.begin(); + if (i == inner_object.end()) { + std::cout << "empty object" << std::endl; + continue; + } else { + for (; i != inner_object.end(); ++i) { + auto inner_field = *i; + std::cout << '"' << inner_field.key() + << "\" : " << inner_field.value() << ", "; + // You can get proper std::string_view for the key and value with: + // std::string_view inner_key = field.unescaped_key(); + // std::string_view value_str = field.value(); + } + } + std::cout << std::endl; + } + // You can break here if you only want just the first element. + // break; + } + } + TEST_SUCCEED(); +} + bool number_tests() { TEST_START(); ondemand::parser parser; @@ -1457,6 +1529,7 @@ bool run() { && number_tests() && current_location_tape_error_with_except() && examplecrt() + && examplecrt_realloc() #endif ; }