From b8426584fc0f5a2f74c2d0dd67adea577d47aa7d Mon Sep 17 00:00:00 2001 From: John Keiser Date: Sat, 19 Dec 2020 10:16:22 -0800 Subject: [PATCH] Make field lookup order-insensitive. --- .../generic/ondemand/json_iterator-inl.h | 8 ++ .../simdjson/generic/ondemand/json_iterator.h | 3 + .../generic/ondemand/token_iterator-inl.h | 4 + .../generic/ondemand/token_iterator.h | 5 + .../generic/ondemand/value_iterator-inl.h | 105 ++++++++++++++++-- tests/ondemand/ondemand_basictests.cpp | 24 ++-- 6 files changed, 125 insertions(+), 24 deletions(-) diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h index c61925e51..30c2e4ddf 100644 --- a/include/simdjson/generic/ondemand/json_iterator-inl.h +++ b/include/simdjson/generic/ondemand/json_iterator-inl.h @@ -156,6 +156,14 @@ simdjson_really_inline error_code json_iterator::report_error(error_code _error, return error; } +simdjson_really_inline const uint32_t *json_iterator::checkpoint() const noexcept { + return token.checkpoint(); +} +simdjson_really_inline void json_iterator::restore_checkpoint(const uint32_t *target_checkpoint) noexcept { + token.restore_checkpoint(target_checkpoint); +} + + simdjson_really_inline error_code json_iterator::optional_error(error_code _error, const char *message) noexcept { SIMDJSON_ASSUME(_error == INCORRECT_TYPE || _error == NO_SUCH_FIELD); logger::log_error(*this, message); diff --git a/include/simdjson/generic/ondemand/json_iterator.h b/include/simdjson/generic/ondemand/json_iterator.h index 8d4c4da35..944a99593 100644 --- a/include/simdjson/generic/ondemand/json_iterator.h +++ b/include/simdjson/generic/ondemand/json_iterator.h @@ -163,6 +163,9 @@ public: template simdjson_warn_unused simdjson_really_inline bool peek_to_buffer(uint8_t (&tmpbuf)[N]) noexcept; template simdjson_warn_unused simdjson_really_inline bool advance_to_buffer(uint8_t (&tmpbuf)[N]) noexcept; + simdjson_really_inline const uint32_t *checkpoint() const noexcept; + simdjson_really_inline void restore_checkpoint(const uint32_t *target_checkpoint) noexcept; + protected: simdjson_really_inline json_iterator(ondemand::parser *parser) noexcept; diff --git a/include/simdjson/generic/ondemand/token_iterator-inl.h b/include/simdjson/generic/ondemand/token_iterator-inl.h index c16d324ea..0398e2221 100644 --- a/include/simdjson/generic/ondemand/token_iterator-inl.h +++ b/include/simdjson/generic/ondemand/token_iterator-inl.h @@ -43,6 +43,10 @@ simdjson_really_inline const uint32_t *token_iterator::checkpoint() const noexce return index; } +simdjson_really_inline void token_iterator::restore_checkpoint(const uint32_t *target_checkpoint) noexcept { + index = target_checkpoint; +} + } // namespace ondemand } // namespace SIMDJSON_IMPLEMENTATION } // namespace simdjson diff --git a/include/simdjson/generic/ondemand/token_iterator.h b/include/simdjson/generic/ondemand/token_iterator.h index 76869fc5b..2db1fa7b3 100644 --- a/include/simdjson/generic/ondemand/token_iterator.h +++ b/include/simdjson/generic/ondemand/token_iterator.h @@ -54,6 +54,11 @@ public: */ simdjson_really_inline const uint32_t *checkpoint() const noexcept; + /** + * Reset to a previously saved index. + */ + simdjson_really_inline void restore_checkpoint(const uint32_t *target_checkpoint) noexcept; + // NOTE: we don't support a full C++ iterator interface, because we expect people to make // different calls to advance the iterator based on *their own* state. diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 9dcc0c5af..be187f6ea 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -51,37 +51,124 @@ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator } } -/** - * Find the field with the given key. May be used in place of ++. - */ simdjson_warn_unused simdjson_really_inline simdjson_result value_iterator::find_field_raw(const std::string_view key) noexcept { - if (!is_open()) { return false; } - - // Unless this is the first field, we need to advance past the , and check for } error_code error; bool has_value; + + // + // Initially, the object can be in one of a few different places: + // + // 1. The start of the object, at the first field: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2, index 1) + // ``` + // if (at_first_field()) { + // If we're at the beginning of the object, we definitely have a field has_value = true; + + // 2. When a previous search did not yield a value or the object is empty: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // { } + // ^ (depth 0, index 2) + // ``` + // + } else if (!is_open()) { + has_value = false; + + // 3. When a previous search found a field or an iterator yielded a value: + // + // ``` + // // When a field was not fully consumed (or not even touched at all) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2) + // // When a field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // When the last field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // ``` + // } else { + // Finish the previous value and see if , or } is next if ((error = skip_child() )) { abandon(); return error; } if ((error = has_next_field().get(has_value) )) { abandon(); return error; } } + + // After initial processing, we will be in one of two states: + // + // ``` + // // At the beginning of a field + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // At the end of the object + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // ``` + // + + // First, we scan from that point to the end. + // If we don't find a match, we loop back around, and scan from the beginning to that point. + const uint32_t *search_start = _json_iter->checkpoint(); + + // Next, we find a match starting from the current position. while (has_value) { - // Get the key + SIMDJSON_ASSUME( _json_iter->_depth == _depth + 1 ); // We must be at the start of a field + + // Get the key and colon, stopping at the value. raw_json_string actual_key; if ((error = field_key().get(actual_key) )) { abandon(); return error; }; if ((error = field_value() )) { abandon(); return error; } - // Check if it matches + // If it matches, stop and return if (actual_key == key) { logger::log_event(*this, "match", key, -2); return true; } + + // No match: skip the value and see if , or } is next logger::log_event(*this, "no match", key, -2); - SIMDJSON_TRY( skip_child() ); // Skip the value entirely + SIMDJSON_TRY( skip_child() ); if ((error = has_next_field().get(has_value) )) { abandon(); return error; } } + // If we reach the end without finding a match, search the rest of the fields starting at the + // beginning of the object. + // (We have already run through the object before, so we've already validated its structure. We + // don't check errors in this bit.) + _json_iter->restore_checkpoint(_start_index + 1); + _json_iter->descend_to(_depth); + + has_value = started_object(); + while (_json_iter->checkpoint() < search_start) { + SIMDJSON_ASSUME(has_value); // we should reach search_start before ever reaching the end of the object + SIMDJSON_ASSUME( _json_iter->_depth == _depth + 1 ); // We must be at the start of a field + + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + error = field_key().get(actual_key); SIMDJSON_ASSUME(!error); + error = field_value(); SIMDJSON_ASSUME(!error); + + // If it matches, stop and return + if (actual_key == key) { + logger::log_event(*this, "match", key, -2); + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + SIMDJSON_TRY( skip_child() ); + error = has_next_field().get(has_value); SIMDJSON_ASSUME(!error); + } + // If the loop ended, we're out of fields to look at. return false; } diff --git a/tests/ondemand/ondemand_basictests.cpp b/tests/ondemand/ondemand_basictests.cpp index 86ea4e5a2..dbfd9b002 100644 --- a/tests/ondemand/ondemand_basictests.cpp +++ b/tests/ondemand/ondemand_basictests.cpp @@ -1166,7 +1166,7 @@ namespace dom_api_tests { ASSERT_EQUAL( object["b"].get_uint64().first, 2 ); ASSERT_EQUAL( object["c/d"].get_uint64().first, 3 ); - ASSERT_ERROR( object["a"], NO_SUCH_FIELD ); + ASSERT_EQUAL( object["a"].get_uint64().first, 1 ); ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); return true; })); @@ -1178,7 +1178,7 @@ namespace dom_api_tests { ASSERT_EQUAL( object["b"].get_uint64().first, 2 ); ASSERT_EQUAL( object["c/d"].get_uint64().first, 3 ); - ASSERT_ERROR( object["a"], NO_SUCH_FIELD ); + ASSERT_EQUAL( object["a"].get_uint64().first, 1 ); ASSERT_ERROR( object["d"], NO_SUCH_FIELD ); return true; })); @@ -1189,7 +1189,7 @@ namespace dom_api_tests { ASSERT_EQUAL( doc["b"].get_uint64().first, 2 ); ASSERT_EQUAL( doc["c/d"].get_uint64().first, 3 ); - ASSERT_ERROR( doc["a"], NO_SUCH_FIELD ); + ASSERT_EQUAL( doc["a"].get_uint64().first, 1 ); ASSERT_ERROR( doc["d"], NO_SUCH_FIELD ); return true; })); @@ -1198,7 +1198,7 @@ namespace dom_api_tests { ASSERT_EQUAL( doc_result["b"].get_uint64().first, 2 ); ASSERT_EQUAL( doc_result["c/d"].get_uint64().first, 3 ); - ASSERT_ERROR( doc_result["a"], NO_SUCH_FIELD ); + ASSERT_EQUAL( doc_result["a"].get_uint64().first, 1 ); ASSERT_ERROR( doc_result["d"], NO_SUCH_FIELD ); return true; })); @@ -1437,19 +1437,13 @@ namespace ordering_tests { double z{0}; for (ondemand::object point_object : doc["coordinates"]) { z += double(point_object["z"]); - try { - x += double(point_object["x"]); - return false; - } catch(simdjson_error&) {} - try { - y += double(point_object["y"]); - return false; - } catch(simdjson_error&) {} + x += double(point_object["x"]); + y += double(point_object["y"]); } - return (x == 0) && (y == 0) && (z == 3.3); + return (x == 1.1) && (y == 2.2) && (z == 3.3); } - bool robust_order() { + bool foreach_lookup() { TEST_START(); ondemand::parser parser{}; auto doc = parser.iterate(json); @@ -1472,7 +1466,7 @@ namespace ordering_tests { #if SIMDJSON_EXCEPTIONS in_order() && out_of_order() && - robust_order() && + foreach_lookup() && #endif true; }