diff --git a/include/simdjson/generic/ondemand/json_iterator-inl.h b/include/simdjson/generic/ondemand/json_iterator-inl.h index 076830a93..48962b7c5 100644 --- a/include/simdjson/generic/ondemand/json_iterator-inl.h +++ b/include/simdjson/generic/ondemand/json_iterator-inl.h @@ -24,6 +24,9 @@ simdjson_inline json_iterator::json_iterator(json_iterator &&other) noexcept _depth{other._depth}, _root{other._root}, _streaming{other._streaming} +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + , _allow_incomplete_json{other._allow_incomplete_json} +#endif { other.parser = nullptr; } @@ -35,6 +38,9 @@ simdjson_inline json_iterator &json_iterator::operator=(json_iterator &&other) n _depth = other._depth; _root = other._root; _streaming = other._streaming; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + _allow_incomplete_json = other._allow_incomplete_json; +#endif other.parser = nullptr; return *this; } @@ -61,7 +67,8 @@ simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parse _string_buf_loc{parser->string_buf.get()}, _depth{1}, _root{parser->implementation->structural_indexes.get()}, - _streaming{streaming} + _streaming{streaming}, + _allow_incomplete_json{true} { logger::log_headers(); @@ -193,6 +200,17 @@ simdjson_inline bool json_iterator::streaming() const noexcept { return _streaming; } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON +simdjson_inline bool json_iterator::allow_incomplete_json() const noexcept { + return _allow_incomplete_json; +} + +simdjson_inline size_t json_iterator::remaining_input_length(const uint8_t *json) const noexcept { + const uint8_t *end = token.buf + parser->_document_len; + return json < end ? size_t(end - json) : 0; +} +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline token_position json_iterator::root_position() const noexcept { return _root; } @@ -450,4 +468,4 @@ simdjson_inline simdjson_result parser::iterate(p if (!json.has_sufficient_padding()) { return INSUFFICIENT_PADDING; } json.remove_utf8_bom(); + _document_len = json.length(); // Allocate if needed if (capacity() < json.length() || !string_buf) { @@ -71,6 +72,7 @@ simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_a if (!json.has_sufficient_padding()) { return INSUFFICIENT_PADDING; } json.remove_utf8_bom(); + _document_len = json.length(); // Allocate if needed if (capacity() < json.length() || !string_buf) { @@ -312,4 +314,4 @@ simdjson_inline simdjson_result::simd } // namespace simdjson -#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H \ No newline at end of file +#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H diff --git a/include/simdjson/generic/ondemand/parser.h b/include/simdjson/generic/ondemand/parser.h index e765f0d0e..30819efe6 100644 --- a/include/simdjson/generic/ondemand/parser.h +++ b/include/simdjson/generic/ondemand/parser.h @@ -431,6 +431,7 @@ private: size_t _capacity{0}; size_t _max_capacity; size_t _max_depth{DEFAULT_MAX_DEPTH}; + size_t _document_len{0}; std::unique_ptr string_buf{}; #if SIMDJSON_DEVELOPMENT_CHECKS diff --git a/include/simdjson/generic/ondemand/value_iterator-inl.h b/include/simdjson/generic/ondemand/value_iterator-inl.h index 154f7fdec..41e30f939 100644 --- a/include/simdjson/generic/ondemand/value_iterator-inl.h +++ b/include/simdjson/generic/ondemand/value_iterator-inl.h @@ -15,6 +15,27 @@ namespace simdjson { namespace SIMDJSON_IMPLEMENTATION { namespace ondemand { +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON +simdjson_inline bool raw_json_string_is_quote_terminated(const uint8_t *json, uint32_t max_len) noexcept { + bool escaping{false}; + for (uint32_t i = 1; i < max_len; i++) { + switch (json[i]) { + case '"': + if (!escaping) { return true; } + escaping = false; + break; + case '\\': + escaping = !escaping; + break; + default: + escaping = false; + break; + } + } + return false; +} +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline value_iterator::value_iterator( json_iterator *json_iter, depth_t depth, @@ -530,6 +551,15 @@ simdjson_warn_unused simdjson_inline simdjson_result value_ite simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_raw_json_string() noexcept { auto json = peek_scalar("string"); if (*json != '"') { return incorrect_type_error("Not a string"); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + if (_json_iter->allow_incomplete_json()) { + const size_t remaining_input_length = _json_iter->remaining_input_length(json); + const uint32_t max_len = remaining_input_length < peek_start_length() ? uint32_t(remaining_input_length) : peek_start_length(); + if (!raw_json_string_is_quote_terminated(json, max_len)) { + return STRING_ERROR; + } + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON advance_scalar("string"); return raw_json_string(json+1); } @@ -677,6 +707,15 @@ simdjson_warn_unused simdjson_inline simdjson_result value_iter auto json = peek_scalar("string"); if (*json != '"') { return incorrect_type_error("Not a string"); } if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + if (_json_iter->allow_incomplete_json()) { + const size_t remaining_input_length = _json_iter->remaining_input_length(json); + const uint32_t max_len = remaining_input_length < peek_root_length() ? uint32_t(remaining_input_length) : peek_root_length(); + if (!raw_json_string_is_quote_terminated(json, max_len)) { + return STRING_ERROR; + } + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON advance_scalar("string"); return raw_json_string(json+1); } @@ -1104,4 +1143,4 @@ simdjson_inline simdjson_result