From 6bed34ad615a6ea9babdd59d48dbb89b8fc5d1ae Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Sat, 21 Aug 2021 10:23:59 -0400 Subject: [PATCH] This exposes 'reset' for object and array instances. (#1696) * This exposes 'rewind' for object and array instances. * Putting really_inline back to count_elements() * Update array.h * Adding empty array rewind. * Adds "is_empty" method to arrays. * More fragmentation. * Tweaking implementation. * Fixing issue with get_value() on document instances. * Changing the name of the new rewind functions to reset. --- include/simdjson/generic/ondemand/array-inl.h | 15 +++ include/simdjson/generic/ondemand/array.h | 29 ++++- .../simdjson/generic/ondemand/document-inl.h | 17 +-- .../simdjson/generic/ondemand/object-inl.h | 22 ++++ include/simdjson/generic/ondemand/object.h | 23 ++++ tests/ondemand/ondemand_array_tests.cpp | 108 ++++++++++++++++- tests/ondemand/ondemand_object_tests.cpp | 109 ++++++++++++++++++ tests/ondemand/ondemand_readme_examples.cpp | 25 ++++ 8 files changed, 331 insertions(+), 17 deletions(-) diff --git a/include/simdjson/generic/ondemand/array-inl.h b/include/simdjson/generic/ondemand/array-inl.h index 1829b0ebb..f71738ed3 100644 --- a/include/simdjson/generic/ondemand/array-inl.h +++ b/include/simdjson/generic/ondemand/array-inl.h @@ -101,6 +101,17 @@ simdjson_really_inline simdjson_result array::count_elements() & noexcep return count; } +simdjson_really_inline simdjson_result array::is_empty() & noexcept { + bool is_not_empty; + auto error = iter.reset_array().get(is_not_empty); + if(error) { return error; } + return !is_not_empty; +} + +inline simdjson_result array::reset() & noexcept { + return iter.reset_array(); +} + inline simdjson_result array::at_pointer(std::string_view json_pointer) noexcept { if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; } json_pointer = json_pointer.substr(1); @@ -179,6 +190,10 @@ simdjson_really_inline simdjson_result simdjson_result simdjson_result::is_empty() & noexcept { + if (error()) { return error(); } + return first.is_empty(); +} simdjson_really_inline simdjson_result simdjson_result::at(size_t index) noexcept { if (error()) { return error(); } return first.at(index); diff --git a/include/simdjson/generic/ondemand/array.h b/include/simdjson/generic/ondemand/array.h index db129b076..20d36e7a3 100644 --- a/include/simdjson/generic/ondemand/array.h +++ b/include/simdjson/generic/ondemand/array.h @@ -41,9 +41,32 @@ public: * beginning as if it had never been accessed. If the JSON is malformed (e.g., * there is a missing comma), then an error is returned and it is no longer * safe to continue. + * + * To check that an array is empty, it is more performant to use + * the is_empty() method. */ simdjson_really_inline simdjson_result count_elements() & noexcept; - + /** + * This method scans the beginning of the array and checks whether the + * array is empty. + * The runtime complexity is constant time. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + simdjson_really_inline simdjson_result is_empty() & noexcept; + /** + * Reset the iterator so that we are pointing back at the + * beginning of the array. You should still consume values only once even if you + * can iterate through the array more than once. If you unescape a string + * within the array more than once, you have unsafe code. Note that rewinding + * an array means that you may need to reparse it anew: it is not a free + * operation. + * + * @returns true if the array contains some elements (not empty) + */ + inline simdjson_result reset() & noexcept; /** * Get the value associated with the given JSON pointer. We use the RFC 6901 * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node @@ -158,7 +181,9 @@ public: simdjson_really_inline simdjson_result begin() noexcept; simdjson_really_inline simdjson_result end() noexcept; - simdjson_really_inline simdjson_result count_elements() & noexcept; + inline simdjson_result count_elements() & noexcept; + inline simdjson_result is_empty() & noexcept; + inline simdjson_result reset() & noexcept; simdjson_really_inline simdjson_result at(size_t index) noexcept; simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; }; diff --git a/include/simdjson/generic/ondemand/document-inl.h b/include/simdjson/generic/ondemand/document-inl.h index b7e6f1d9c..37917309c 100644 --- a/include/simdjson/generic/ondemand/document-inl.h +++ b/include/simdjson/generic/ondemand/document-inl.h @@ -40,20 +40,9 @@ simdjson_really_inline simdjson_result document::get_value() noexcept { // gets called. iter.assert_at_document_depth(); switch (*iter.peek()) { - case '[': { - array result; - SIMDJSON_TRY( get_array().get(result) ); - iter._depth = 1 ; /* undoing the potential increment so we go back at the doc depth.*/ - iter.assert_at_document_depth(); - return value(result.iter); - } - case '{': { - object result; - SIMDJSON_TRY( get_object().get(result) ); - iter._depth = 1 ; /* undoing the potential increment so we go back at the doc depth.*/ - iter.assert_at_document_depth(); - return value(result.iter); - } + case '[': + case '{': + return value(get_root_value_iterator()); default: // Unfortunately, scalar documents are a special case in simdjson and they cannot // be safely converted to value instances. diff --git a/include/simdjson/generic/ondemand/object-inl.h b/include/simdjson/generic/ondemand/object-inl.h index 0bd49e11c..aa788a6bd 100644 --- a/include/simdjson/generic/ondemand/object-inl.h +++ b/include/simdjson/generic/ondemand/object-inl.h @@ -139,6 +139,18 @@ inline simdjson_result object::at_pointer(std::string_view json_pointer) return child; } + +simdjson_really_inline simdjson_result object::is_empty() & noexcept { + bool is_not_empty; + auto error = iter.reset_object().get(is_not_empty); + if(error) { return error; } + return !is_not_empty; +} + +simdjson_really_inline simdjson_result object::reset() & noexcept { + return iter.reset_object(); +} + } // namespace ondemand } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson @@ -188,4 +200,14 @@ simdjson_really_inline simdjson_result return first.at_pointer(json_pointer); } +inline simdjson_result simdjson_result::reset() noexcept { + if (error()) { return error(); } + return first.reset(); +} + +inline simdjson_result simdjson_result::is_empty() noexcept { + if (error()) { return error(); } + return first.is_empty(); +} + } // namespace simdjson diff --git a/include/simdjson/generic/ondemand/object.h b/include/simdjson/generic/ondemand/object.h index 8164646e1..bac28867d 100644 --- a/include/simdjson/generic/ondemand/object.h +++ b/include/simdjson/generic/ondemand/object.h @@ -110,6 +110,26 @@ public: */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Reset the iterator so that we are pointing back at the + * beginning of the object. You should still consume values only once even if you + * can iterate through the object more than once. If you unescape a string within + * the object more than once, you have unsafe code. Note that rewinding an object + * means that you may need to reparse it anew: it is not a free operation. + * + * @returns true if the object contains some elements (not empty) + */ + inline simdjson_result reset() & noexcept; + /** + * This method scans the beginning of the object and checks whether the + * object is empty. + * The runtime complexity is constant time. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + inline simdjson_result is_empty() & noexcept; /** * Consumes the object and returns a string_view instance corresponding to the * object as represented in JSON. It points inside the original byte array containg @@ -159,6 +179,9 @@ public: simdjson_really_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_really_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_really_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + inline simdjson_result reset() noexcept; + inline simdjson_result is_empty() noexcept; + }; } // namespace simdjson diff --git a/tests/ondemand/ondemand_array_tests.cpp b/tests/ondemand/ondemand_array_tests.cpp index 787a667c8..4a87d67e1 100644 --- a/tests/ondemand/ondemand_array_tests.cpp +++ b/tests/ondemand/ondemand_array_tests.cpp @@ -308,7 +308,30 @@ namespace array_tests { } return true; })); + SUBTEST("ondemand::array-rewind", test_ondemand_doc(json, [&](auto doc_result) { + ondemand::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + size_t i = 0; + for (auto value : array) { (void)value; i++; } + ASSERT_EQUAL(i*sizeof(uint64_t), sizeof(expected_value)); + std::vector container(i); // container of size 'i'. + + array.reset(); + i = 0; + for (auto value : array) { + int64_t actual; + ASSERT_SUCCESS( value.get(actual) ); + container[i] = actual; + i++; + } + ASSERT_EQUAL(i * sizeof(int64_t), sizeof(expected_value)); + for(size_t j = 0; j < sizeof(expected_value)/sizeof(int64_t); j++) { + ASSERT_EQUAL(container[j], expected_value[j]); + } + return true; + })); SUBTEST("simdjson_result", test_ondemand_doc(json, [&](auto doc_result) { simdjson_result array = doc_result.get_array(); ASSERT_RESULT( doc_result.type(), json_type::array ); @@ -336,7 +359,71 @@ namespace array_tests { })); TEST_SUCCEED(); } - + bool empty_rewind() { + TEST_START(); + const auto json = R"( [] )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + for(simdjson_unused auto i : arr) { + TEST_FAIL("should be empty?"); + } + arr.reset(); + for(simdjson_unused auto i : arr) { + TEST_FAIL("should be empty?"); + } + TEST_SUCCEED(); + } + bool count_empty(simdjson::ondemand::array arr) { + size_t count; + ASSERT_SUCCESS(arr.count_elements().get(count)); + ASSERT_EQUAL(count, 0); + bool is_empty; + ASSERT_SUCCESS(arr.is_empty().get(is_empty)); + ASSERT_TRUE(is_empty); + return true; + } + bool value_to_array(simdjson::ondemand::value val) { + ondemand::json_type t; + ASSERT_SUCCESS(val.type().get(t)); + ASSERT_EQUAL(t, ondemand::json_type::array); + simdjson::ondemand::array arr; + ASSERT_SUCCESS(val.get_array().get(arr)); + if(count_empty(arr) != true) { return false; } + return true; + } + bool empty_rewind_convoluted() { + TEST_START(); + const auto json = R"( [] )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(!value_to_array(val)) { return false; } + TEST_SUCCEED(); + } +#if SIMDJSON_EXCEPTIONS + bool value_to_array_except(simdjson::ondemand::value val) { + ondemand::json_type t = val.type(); + ASSERT_EQUAL(t, ondemand::json_type::array); + if(count_empty(simdjson::ondemand::array(val)) != true) { return false; } + return true; + } + bool empty_rewind_convoluted_with_exceptions() { + TEST_START(); + const auto json = R"( [] )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(value_to_array_except(val) != true) { return false; } + TEST_SUCCEED(); + } +#endif bool iterate_array() { TEST_START(); const auto json = R"( [ [ 1, 10, 100 ] ] )"_padded; @@ -591,8 +678,24 @@ namespace array_tests { ASSERT_EQUAL(i, 0); return true; })); + SUBTEST("ondemand::array-rewind", test_ondemand_doc(json, [&](auto doc_result) { + ondemand::array array; + ASSERT_RESULT( doc_result.type(), json_type::array ); + ASSERT_SUCCESS( doc_result.get(array) ); + + size_t i = 0; + for (auto value : array) { (void) value; i++; } + ASSERT_EQUAL(i, 0); + + array.reset(); + i = 0; + for (auto value : array) { (void) value; i++; } + ASSERT_EQUAL(i, 0); + return true; + })); TEST_SUCCEED(); } + #if SIMDJSON_EXCEPTIONS bool iterate_array_exception() { @@ -639,6 +742,8 @@ namespace array_tests { bool run() { return + empty_rewind_convoluted() && + empty_rewind() && iterate_empty_array_count() && iterate_sub_array_count() && iterate_complex_array_count() && @@ -650,6 +755,7 @@ namespace array_tests { iterate_empty_array() && iterate_array_partial_children() && #if SIMDJSON_EXCEPTIONS + empty_rewind_convoluted_with_exceptions() && iterate_array_exception() && #endif // SIMDJSON_EXCEPTIONS true; diff --git a/tests/ondemand/ondemand_object_tests.cpp b/tests/ondemand/ondemand_object_tests.cpp index 25bb98f43..740fac30f 100644 --- a/tests/ondemand/ondemand_object_tests.cpp +++ b/tests/ondemand/ondemand_object_tests.cpp @@ -237,6 +237,29 @@ namespace object_tests { ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); return true; })); + SUBTEST("ondemand::object-rewind", test_ondemand_doc(json, [&](auto doc_result) { + ondemand::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + object.reset(); + i = 0; + for (auto field : object) { + ASSERT_SUCCESS( field.error() ); + ASSERT_EQUAL( field.key().value_unsafe(), expected_key[i]); + ASSERT_EQUAL( field.value().get_uint64().value_unsafe(), expected_value[i] ); + i++; + } + ASSERT_EQUAL( i*sizeof(uint64_t), sizeof(expected_value) ); + return true; + })); SUBTEST("simdjson_result", test_ondemand_doc(json, [&](auto doc_result) { simdjson_result object_result = doc_result.get_object(); size_t i = 0; @@ -590,12 +613,96 @@ namespace object_tests { ASSERT_EQUAL( i, 0 ); return true; })); + SUBTEST("ondemand::object-rewind", test_ondemand_doc(json, [&](auto doc_result) { + ondemand::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + object.reset(); + i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + return true; + })); + SUBTEST("ondemand::object-rewind", test_ondemand_doc(json, [&](auto doc_result) { + ondemand::object object; + ASSERT_RESULT( doc_result.type(), json_type::object ); + ASSERT_SUCCESS( doc_result.get(object) ); + size_t i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + object.reset(); + i = 0; + for (auto field : object) { + (void)field; + i++; + } + ASSERT_EQUAL( i, 0 ); + return true; + })); TEST_SUCCEED(); } #endif // SIMDJSON_EXCEPTIONS + bool empty(simdjson::ondemand::object obj) { + bool is_empty; + ASSERT_SUCCESS(obj.is_empty().get(is_empty)); + ASSERT_TRUE(is_empty); + return true; + } + bool value_to_object(simdjson::ondemand::value val) { + ondemand::json_type t; + ASSERT_SUCCESS(val.type().get(t)); + ASSERT_EQUAL(t, ondemand::json_type::object); + simdjson::ondemand::object obj; + ASSERT_SUCCESS(val.get_object().get(obj)); + if(empty(obj) != true) { return false; } + return true; + } + bool empty_rewind_convoluted() { + TEST_START(); + const auto json = R"( {} )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(!value_to_object(val)) { return false; } + TEST_SUCCEED(); + } +#if SIMDJSON_EXCEPTIONS + bool value_to_object_except(simdjson::ondemand::value val) { + ondemand::json_type t = val.type(); + ASSERT_EQUAL(t, ondemand::json_type::object); + if(empty(simdjson::ondemand::object(val)) != true) { return false; } + return true; + } + bool empty_rewind_convoluted_with_exceptions() { + TEST_START(); + const auto json = R"( {} )"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::value val; + ASSERT_SUCCESS(doc.get_value().get(val)); + if(value_to_object_except(val) != true) { return false; } + TEST_SUCCEED(); + } +#endif + bool run() { return value_search_unescaped_key() && @@ -612,8 +719,10 @@ namespace object_tests { iterate_empty_object() && iterate_object_partial_children() && issue_1480() && + empty_rewind_convoluted() && #if SIMDJSON_EXCEPTIONS iterate_object_exception() && + empty_rewind_convoluted_with_exceptions() && #endif // SIMDJSON_EXCEPTIONS true; } diff --git a/tests/ondemand/ondemand_readme_examples.cpp b/tests/ondemand/ondemand_readme_examples.cpp index 91d21ca73..24a0ac567 100644 --- a/tests/ondemand/ondemand_readme_examples.cpp +++ b/tests/ondemand/ondemand_readme_examples.cpp @@ -305,6 +305,30 @@ bool using_the_parsed_json_rewind() { TEST_SUCCEED(); } + +bool using_the_parsed_json_rewind_array() { + TEST_START(); + + ondemand::parser parser; + auto cars_json = R"( [ + { "make": "Toyota", "model": "Camry", "year": 2018, "tire_pressure": [ 40.1, 39.9, 37.7, 40.4 ] }, + { "make": "Kia", "model": "Soul", "year": 2012, "tire_pressure": [ 30.1, 31.0, 28.6, 28.7 ] }, + { "make": "Toyota", "model": "Tercel", "year": 1999, "tire_pressure": [ 29.8, 30.0, 30.2, 30.5 ] } + ] )"_padded; + + auto doc = parser.iterate(cars_json); + ondemand::array arr = doc.get_array(); + size_t count = 0; + for (simdjson_unused ondemand::object car : arr) { + if(car["make"] == "Toyota") { count++; } + } + std::cout << "We have " << count << " Toyota cars.\n"; + arr.reset(); + for (ondemand::object car : arr) { + cout << "Make/Model: " << std::string_view(car["make"]) << "/" << std::string_view(car["model"]) << endl; + } + TEST_SUCCEED(); +} bool using_the_parsed_json_4() { TEST_START(); @@ -738,6 +762,7 @@ int main() { && json_array_count_complex() && json_array_count() && using_the_parsed_json_rewind() + && using_the_parsed_json_rewind_array() && basics_2() && using_the_parsed_json_1() && using_the_parsed_json_2()