From cfd337bb50dbd9c102657464291924fcf6412ae9 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 30 Mar 2023 10:47:13 -0400 Subject: [PATCH] This PR adds an 'at_end()' method. (#1978) * This PR adds an 'at_end()' method. * Adding 1111 } * Tweaking test. --- benchmark/json2msgpack/simdjson_ondemand.h | 3 + doc/basics.md | 38 ++++++++---- .../simdjson/generic/ondemand/document-inl.h | 15 ++++- include/simdjson/generic/ondemand/document.h | 10 ++- .../generic/ondemand/json_iterator-inl.h | 2 +- .../simdjson/generic/ondemand/json_iterator.h | 2 +- .../generic/ondemand/value_iterator-inl.h | 10 +++ src/internal/error_tables.cpp | 62 +++++++++---------- tests/ondemand/ondemand_array_tests.cpp | 15 +++++ .../ondemand_document_stream_tests.cpp | 15 +++++ tests/ondemand/ondemand_object_tests.cpp | 17 ++++- tests/ondemand/ondemand_readme_examples.cpp | 15 +++++ 12 files changed, 156 insertions(+), 48 deletions(-) diff --git a/benchmark/json2msgpack/simdjson_ondemand.h b/benchmark/json2msgpack/simdjson_ondemand.h index 83f989cd9..6407a62fc 100644 --- a/benchmark/json2msgpack/simdjson_ondemand.h +++ b/benchmark/json2msgpack/simdjson_ondemand.h @@ -100,6 +100,9 @@ simdjson2msgpack::to_msgpack(const simdjson::padded_string &json, recursive_processor_ref(val); #endif } + if (!doc.at_end()) { + throw "There are unexpectedly tokens after the end of the json in the json2msgpack sample data"; + } return std::string_view(reinterpret_cast(buf), size_t(buff - buf)); } diff --git a/doc/basics.md b/doc/basics.md index c9168f818..5337c70a1 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -23,6 +23,7 @@ An overview of what you need to know to use simdjson, with examples. - [Disabling Exceptions](#disabling-exceptions) - [Exceptions](#exceptions) - [Current location in document](#current-location-in-document) + - [Checking for trailing content](#checking-for-trailing-content) - [Rewinding](#rewinding) - [Direct Access to the Raw String](#direct-access-to-the-raw-string) - [Newline-Delimited JSON (ndjson) and JSON lines](#newline-delimited-json-ndjson-and-json-lines) @@ -1152,18 +1153,6 @@ if (error) { } ``` - - auto broken_json = R"( {"double": 13.06, false, "integer": -343} )"_padded; - ondemand::parser parser; - ondemand::document doc; - ASSERT_SUCCESS(parser.iterate(broken_json).get(doc)); - const char * ptr; - int64_t i; - ASSERT_ERROR(doc["integer"].get_int64().get(i), TAPE_ERROR); - ASSERT_SUCCESS(doc.current_location().get(ptr)); - std::string expected = "false, \"integer\": -343} "; - ASSERT_EQUAL(std::string(ptr,expected.size()), expected); - You may also use `current_location()` with exceptions as follows: ```c++ @@ -1236,6 +1225,31 @@ contains control characters that must be escaped and UNCLOSED_STRING if there is an unclosed string in the document. We do not provide location information for these errors. +### Checking for trailing content + +The parser validates all parsed content, but your code may exhaust the content while +not having processed the entire document. Thus, as a final optional step, you may +call `at_end()` on the document instance. If it returns `false`, then you may +conclude that you have trailing content and that your document is not valid JSON. +You may then use `doc.current_location()` to obtain a pointer to the start of the trailing +content. + +```C++ + auto json = R"([1, 2] foo ])"_padded; + ondemand::parser parser; + ondemand::document doc = parser.iterate(json); + ondemand::array array = doc.get_array(); + for (uint64_t values : array) { + std::cout << values << std::endl; + } + if(!doc.at_end()) { + std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl; + } +``` + +The `at_end()` method is equivalent to `doc.current_location().error() == simdjson::SUCCESS` but +more convenient. + Rewinding ---------- diff --git a/include/simdjson/generic/ondemand/document-inl.h b/include/simdjson/generic/ondemand/document-inl.h index 75a531978..691616d81 100644 --- a/include/simdjson/generic/ondemand/document-inl.h +++ b/include/simdjson/generic/ondemand/document-inl.h @@ -20,7 +20,7 @@ inline std::string document::to_debug_string() noexcept { return iter.to_string(); } -inline simdjson_result document::current_location() noexcept { +inline simdjson_result document::current_location() const noexcept { return iter.current_location(); } @@ -28,6 +28,11 @@ inline int32_t document::current_depth() const noexcept { return iter.depth(); } +inline bool document::at_end() const noexcept { + return iter.at_end(); +} + + inline bool document::is_alive() noexcept { return iter.is_alive(); } @@ -50,12 +55,14 @@ simdjson_inline simdjson_result document::get_value() noexcept { iter.assert_at_document_depth(); switch (*iter.peek()) { case '[': { + // The following lines check that the document ends with ]. auto value_iterator = get_root_value_iterator(); auto error = value_iterator.check_root_array(); if(error) { return error; } return value(get_root_value_iterator()); } case '{': { + // The following lines would check that the document ends with }. auto value_iterator = get_root_value_iterator(); auto error = value_iterator.check_root_object(); if(error) { return error; } @@ -505,6 +512,12 @@ simdjson_inline simdjson_result simdjson_result::at_end() const noexcept { + if (error()) { return error(); } + return first.at_end(); +} + + simdjson_inline int32_t simdjson_result::current_depth() const noexcept { if (error()) { return error(); } return first.current_depth(); diff --git a/include/simdjson/generic/ondemand/document.h b/include/simdjson/generic/ondemand/document.h index 63d33a53b..84649ddb5 100644 --- a/include/simdjson/generic/ondemand/document.h +++ b/include/simdjson/generic/ondemand/document.h @@ -511,7 +511,14 @@ public: /** * Returns the current location in the document if in bounds. */ - inline simdjson_result current_location() noexcept; + inline simdjson_result current_location() const noexcept; + + /** + * Returns true if this document has been fully parsed. + * If you have consumed the whole document and at_end() returns + * false, then there may be trailing content. + */ + inline bool at_end() const noexcept; /** * Returns the current depth in the document if in bounds. @@ -720,6 +727,7 @@ public: simdjson_inline simdjson_result is_scalar() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool at_end() const noexcept; simdjson_inline bool is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h index 987b56290..cfa3ecbb7 100644 --- a/include/simdjson/generic/ondemand/json_iterator-inl.h +++ b/include/simdjson/generic/ondemand/json_iterator-inl.h @@ -208,7 +208,7 @@ inline std::string json_iterator::to_string() const noexcept { + std::string(" ]"); } -inline simdjson_result json_iterator::current_location() noexcept { +inline simdjson_result json_iterator::current_location() const noexcept { if (!is_alive()) { // Unrecoverable error if (!at_root()) { return reinterpret_cast(token.peek(-1)); diff --git a/include/simdjson/generic/ondemand/json_iterator.h b/include/simdjson/generic/ondemand/json_iterator.h index 5b109c175..17d2a093d 100644 --- a/include/simdjson/generic/ondemand/json_iterator.h +++ b/include/simdjson/generic/ondemand/json_iterator.h @@ -261,7 +261,7 @@ public: /** * Returns the current location in the document if in bounds. */ - inline simdjson_result current_location() noexcept; + inline simdjson_result current_location() const noexcept; /** * Updates this json iterator so that it is back at the beginning of the document, diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 40268599f..b2d251b06 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -40,6 +40,11 @@ simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_objec // Note that adding a check for 'streaming' is not expensive since we only have at most // one root element. if ( ! _json_iter->streaming() ) { + // The following lines do not fully protect against garbage content within the + // object: e.g., `{"a":2} foo }`. Users concerned with garbage content should + // call `at_end()` on the document instance at the end of the processing to + // ensure that the processing has finished at the end. + // if (*_json_iter->peek_last() != '}') { _json_iter->abandon(); return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end"); @@ -431,6 +436,11 @@ simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_array // Note that adding a check for 'streaming' is not expensive since we only have at most // one root element. if ( ! _json_iter->streaming() ) { + // The following lines do not fully protect against garbage content within the + // array: e.g., `[1, 2] foo]`. Users concerned with garbage content should + // also call `at_end()` on the document instance at the end of the processing to + // ensure that the processing has finished at the end. + // if (*_json_iter->peek_last() != ']') { _json_iter->abandon(); return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end"); diff --git a/src/internal/error_tables.cpp b/src/internal/error_tables.cpp index 28a139f9f..e6ef21290 100644 --- a/src/internal/error_tables.cpp +++ b/src/internal/error_tables.cpp @@ -4,37 +4,37 @@ namespace simdjson { namespace internal { SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[] { - { SUCCESS, "No error" }, - { CAPACITY, "This parser can't support a document that big" }, - { MEMALLOC, "Error allocating memory, we're most likely out of memory" }, - { TAPE_ERROR, "The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc." }, - { DEPTH_ERROR, "The JSON document was too deep (too many nested objects and arrays)" }, - { STRING_ERROR, "Problem while parsing a string" }, - { T_ATOM_ERROR, "Problem while parsing an atom starting with the letter 't'" }, - { F_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'f'" }, - { N_ATOM_ERROR, "Problem while parsing an atom starting with the letter 'n'" }, - { NUMBER_ERROR, "Problem while parsing a number" }, - { UTF8_ERROR, "The input is not valid UTF-8" }, - { UNINITIALIZED, "Uninitialized" }, - { EMPTY, "Empty: no JSON found" }, - { UNESCAPED_CHARS, "Within strings, some characters must be escaped, we found unescaped characters" }, - { UNCLOSED_STRING, "A string is opened, but never closed." }, - { UNSUPPORTED_ARCHITECTURE, "simdjson does not have an implementation supported by this CPU architecture (perhaps it's a non-SIMD CPU?)." }, - { INCORRECT_TYPE, "The JSON element does not have the requested type." }, - { NUMBER_OUT_OF_RANGE, "The JSON number is too large or too small to fit within the requested type." }, - { INDEX_OUT_OF_BOUNDS, "Attempted to access an element of a JSON array that is beyond its length." }, - { NO_SUCH_FIELD, "The JSON field referenced does not exist in this object." }, - { IO_ERROR, "Error reading the file." }, - { INVALID_JSON_POINTER, "Invalid JSON pointer syntax." }, - { INVALID_URI_FRAGMENT, "Invalid URI fragment syntax." }, - { UNEXPECTED_ERROR, "Unexpected error, consider reporting this problem as you may have found a bug in simdjson" }, - { PARSER_IN_USE, "Cannot parse a new document while a document is still in use." }, - { OUT_OF_ORDER_ITERATION, "Objects and arrays can only be iterated when they are first encountered." }, - { INSUFFICIENT_PADDING, "simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." }, - { INCOMPLETE_ARRAY_OR_OBJECT, "JSON document ended early in the middle of an object or array." }, - { SCALAR_DOCUMENT_AS_VALUE, "A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "}, - { OUT_OF_BOUNDS, "Attempted to access location outside of document."}, - { TRAILING_CONTENT, "Unexpected trailing content in the JSON input."} + { SUCCESS, "SUCCESS: No error" }, + { CAPACITY, "CAPACITY: This parser can't support a document that big" }, + { MEMALLOC, "MEMALLOC: Error allocating memory, we're most likely out of memory" }, + { TAPE_ERROR, "TAPE_ERROR: The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc." }, + { DEPTH_ERROR, "DEPTH_ERROR: The JSON document was too deep (too many nested objects and arrays)" }, + { STRING_ERROR, "STRING_ERROR: Problem while parsing a string" }, + { T_ATOM_ERROR, "T_ATOM_ERROR: Problem while parsing an atom starting with the letter 't'" }, + { F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" }, + { N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" }, + { NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" }, + { UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" }, + { UNINITIALIZED, "UNINITIALIZED: Uninitialized" }, + { EMPTY, "EMPTY: no JSON found" }, + { UNESCAPED_CHARS, "UNESCAPED_CHARS: Within strings, some characters must be escaped, we found unescaped characters" }, + { UNCLOSED_STRING, "UNCLOSED_STRING: A string is opened, but never closed." }, + { UNSUPPORTED_ARCHITECTURE, "UNSUPPORTED_ARCHITECTURE: simdjson does not have an implementation supported by this CPU architecture. Please report this error to the core team as it should never happen." }, + { INCORRECT_TYPE, "INCORRECT_TYPE: The JSON element does not have the requested type." }, + { NUMBER_OUT_OF_RANGE, "NUMBER_OUT_OF_RANGE: The JSON number is too large or too small to fit within the requested type." }, + { INDEX_OUT_OF_BOUNDS, "INDEX_OUT_OF_BOUNDS: Attempted to access an element of a JSON array that is beyond its length." }, + { NO_SUCH_FIELD, "NO_SUCH_FIELD: The JSON field referenced does not exist in this object." }, + { IO_ERROR, "IO_ERROR: Error reading the file." }, + { INVALID_JSON_POINTER, "INVALID_JSON_POINTER: Invalid JSON pointer syntax." }, + { INVALID_URI_FRAGMENT, "INVALID_URI_FRAGMENT: Invalid URI fragment syntax." }, + { UNEXPECTED_ERROR, "UNEXPECTED_ERROR: Unexpected error, consider reporting this problem as you may have found a bug in simdjson" }, + { PARSER_IN_USE, "PARSER_IN_USE: Cannot parse a new document while a document is still in use." }, + { OUT_OF_ORDER_ITERATION, "OUT_OF_ORDER_ITERATION: Objects and arrays can only be iterated when they are first encountered." }, + { INSUFFICIENT_PADDING, "INSUFFICIENT_PADDING: simdjson requires the input JSON string to have at least SIMDJSON_PADDING extra bytes allocated, beyond the string's length. Consider using the simdjson::padded_string class if needed." }, + { INCOMPLETE_ARRAY_OR_OBJECT, "INCOMPLETE_ARRAY_OR_OBJECT: JSON document ended early in the middle of an object or array." }, + { SCALAR_DOCUMENT_AS_VALUE, "SCALAR_DOCUMENT_AS_VALUE: A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "}, + { OUT_OF_BOUNDS, "OUT_OF_BOUNDS: Attempt to access location outside of document."}, + { TRAILING_CONTENT, "TRAILING_CONTENT: Unexpected trailing content in the JSON input."} }; // error_messages[] } // namespace internal diff --git a/tests/ondemand/ondemand_array_tests.cpp b/tests/ondemand/ondemand_array_tests.cpp index 22f4dc65f..0c5b910a9 100644 --- a/tests/ondemand/ondemand_array_tests.cpp +++ b/tests/ondemand/ondemand_array_tests.cpp @@ -6,6 +6,20 @@ using namespace simdjson; namespace array_tests { using namespace std; using simdjson::ondemand::json_type; + bool issue1977() { + TEST_START(); + auto json = R"([1, 2] foo ])"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::array array; + ASSERT_SUCCESS(doc.get_array().get(array)); + for (auto values : array) { + ASSERT_SUCCESS(values); + } + ASSERT_FALSE(doc.at_end()); + TEST_SUCCEED(); + } bool issue1588() { TEST_START(); const auto json = R"({ @@ -830,6 +844,7 @@ namespace array_tests { bool run() { return + issue1977() && issue1876() && issue1742() && empty_rewind_convoluted() && diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp index 9c809a75e..0f4b6224e 100644 --- a/tests/ondemand/ondemand_document_stream_tests.cpp +++ b/tests/ondemand/ondemand_document_stream_tests.cpp @@ -193,6 +193,20 @@ namespace document_stream_tests { TEST_SUCCEED(); } + bool issue1977() { + TEST_START(); + std::string json = R"( 1111 })"; + ondemand::parser odparser; + ondemand::document_stream odstream; + ASSERT_SUCCESS(odparser.iterate_many(json).get(odstream)); + + auto i = odstream.begin(); + for (; i != odstream.end(); ++i) { + ASSERT_TRUE(false); + } + ASSERT_TRUE(i.current_index() == 0); + TEST_SUCCEED(); + } bool issue1683() { TEST_START(); @@ -789,6 +803,7 @@ namespace document_stream_tests { bool run() { return + issue1977() && string_with_trailing() && uint64_with_trailing() && int64_with_trailing() && diff --git a/tests/ondemand/ondemand_object_tests.cpp b/tests/ondemand/ondemand_object_tests.cpp index e1e13a43f..dfdba461e 100644 --- a/tests/ondemand/ondemand_object_tests.cpp +++ b/tests/ondemand/ondemand_object_tests.cpp @@ -7,6 +7,21 @@ namespace object_tests { using namespace std; using simdjson::ondemand::json_type; + bool issue1977() { + TEST_START(); + auto json = R"({"1": 2} foo })"_padded; + ondemand::parser parser; + ondemand::document doc; + ASSERT_SUCCESS(parser.iterate(json).get(doc)); + ondemand::object object; + ASSERT_SUCCESS(doc.get_object().get(object)); + for (auto values : object) { + ASSERT_SUCCESS(values); + } + ASSERT_FALSE(doc.at_end()); + TEST_SUCCEED(); + } + bool issue1745() { TEST_START(); auto json = R"({ @@ -1263,7 +1278,7 @@ namespace object_tests { } bool run() { - return + return issue1977() && #if SIMDJSON_EXCEPTIONS issue1965() && #endif diff --git a/tests/ondemand/ondemand_readme_examples.cpp b/tests/ondemand/ondemand_readme_examples.cpp index 6c847e933..aaf27c006 100644 --- a/tests/ondemand/ondemand_readme_examples.cpp +++ b/tests/ondemand/ondemand_readme_examples.cpp @@ -9,6 +9,19 @@ using error_code=simdjson::error_code; #if SIMDJSON_EXCEPTIONS + bool at_end() { + auto json = R"([1, 2] foo ])"_padded; + ondemand::parser parser; + ondemand::document doc = parser.iterate(json); + ondemand::array array = doc.get_array(); + for (uint64_t values : array) { + std::cout << values << std::endl; + } + if(!doc.at_end()) { + std::cerr << "trailing content at byte index " << doc.current_location() - json.data() << std::endl; + } + TEST_SUCCEED(); + } bool number_tests() { ondemand::parser parser; padded_string docdata = R"([1.0, 3, 1, 3.1415,-13231232,9999999999999999999])"_padded; @@ -978,6 +991,7 @@ bool current_location_tape_error() { TEST_SUCCEED(); } + bool current_location_user_error() { TEST_START(); auto json = R"( [1,2,3] )"_padded; @@ -1208,6 +1222,7 @@ bool example1958() { bool run() { return true #if SIMDJSON_EXCEPTIONS + && at_end() && example1956() && example1958() // && basics_1() // Fails because twitter.json isn't in current directory. Compile test only. && basics_treewalk()