admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Bug fix for get_string() crash when parse incomplete json string (in option disabled my default) (#2720)

Browse Source 
* get_string() crash when parse incomplete json string

* get_string() crash when parse incomplete json string

* Bug fix for RISC-V fail

* Update tests/ondemand/CMakeLists.txt

Co-authored-by: Daniel Lemire <daniel@lemire.me>

* Remove unnecessary changes

---------

Co-authored-by: Daniel Lemire <daniel@lemire.me>
This commit is contained in:
wankun
2026-05-19 05:18:58 +08:00
committed by GitHub
parent 396ca141fd
commit 1ae93a70d4
6 changed files with 97 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/simdjson/generic/ondemand/json_iterator-inl.h
+20 -2
View File
@@ -24,6 +24,9 @@ simdjson_inline json_iterator::json_iterator(json_iterator &&other) noexcept
_depth{other._depth},
_root{other._root},
_streaming{other._streaming}
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
, _allow_incomplete_json{other._allow_incomplete_json}
#endif
{
other.parser = nullptr;
}
@@ -35,6 +38,9 @@ simdjson_inline json_iterator &json_iterator::operator=(json_iterator &&other) n
_depth = other._depth;
_root = other._root;
_streaming = other._streaming;
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
_allow_incomplete_json = other._allow_incomplete_json;
#endif
other.parser = nullptr;
return *this;
}
@@ -61,7 +67,8 @@ simdjson_inline json_iterator::json_iterator(const uint8_t *buf, ondemand::parse
_string_buf_loc{parser->string_buf.get()},
_depth{1},
_root{parser->implementation->structural_indexes.get()},
_streaming{streaming}
_streaming{streaming},
_allow_incomplete_json{true}
{
logger::log_headers();
@@ -193,6 +200,17 @@ simdjson_inline bool json_iterator::streaming() const noexcept {
return _streaming;
}
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
simdjson_inline bool json_iterator::allow_incomplete_json() const noexcept {
return _allow_incomplete_json;
}
simdjson_inline size_t json_iterator::remaining_input_length(const uint8_t *json) const noexcept {
const uint8_t *end = token.buf + parser->_document_len;
return json < end ? size_t(end - json) : 0;
}
#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
simdjson_inline token_position json_iterator::root_position() const noexcept {
return _root;
}
@@ -450,4 +468,4 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::json_iterator
} // namespace simdjson
#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H
#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H
include/simdjson/generic/ondemand/json_iterator.h
+8 -1
View File
@@ -60,6 +60,9 @@ protected:
* value of this attribute.
*/
bool _streaming{false};
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
bool _allow_incomplete_json{false};
#endif
public:
simdjson_inline json_iterator() noexcept = default;
@@ -84,6 +87,10 @@ public:
* start_root_array() and start_root_object().
*/
simdjson_inline bool streaming() const noexcept;
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
simdjson_inline bool allow_incomplete_json() const noexcept;
simdjson_inline size_t remaining_input_length(const uint8_t *json) const noexcept;
#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
/**
* Get the root value iterator
@@ -335,4 +342,4 @@ public:
} // namespace simdjson
#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H
#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H
include/simdjson/generic/ondemand/parser-inl.h
+3 -1
View File
@@ -55,6 +55,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate(p
if (!json.has_sufficient_padding()) { return INSUFFICIENT_PADDING; }
json.remove_utf8_bom();
_document_len = json.length();
// Allocate if needed
if (capacity() < json.length() || !string_buf) {
@@ -71,6 +72,7 @@ simdjson_warn_unused simdjson_inline simdjson_result<document> parser::iterate_a
if (!json.has_sufficient_padding()) { return INSUFFICIENT_PADDING; }
json.remove_utf8_bom();
_document_len = json.length();
// Allocate if needed
if (capacity() < json.length() || !string_buf) {
@@ -312,4 +314,4 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::parser>::simd
} // namespace simdjson
#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H
#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H
include/simdjson/generic/ondemand/parser.h
+1
View File
@@ -431,6 +431,7 @@ private:
size_t _capacity{0};
size_t _max_capacity;
size_t _max_depth{DEFAULT_MAX_DEPTH};
size_t _document_len{0};
std::unique_ptr<uint8_t[]> string_buf{};
#if SIMDJSON_DEVELOPMENT_CHECKS
include/simdjson/generic/ondemand/value_iterator-inl.h
+40 -1
View File
@@ -15,6 +15,27 @@ namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
simdjson_inline bool raw_json_string_is_quote_terminated(const uint8_t *json, uint32_t max_len) noexcept {
bool escaping{false};
for (uint32_t i = 1; i < max_len; i++) {
switch (json[i]) {
case '"':
if (!escaping) { return true; }
escaping = false;
break;
case '\\':
escaping = !escaping;
break;
default:
escaping = false;
break;
}
}
return false;
}
#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
simdjson_inline value_iterator::value_iterator(
json_iterator *json_iter,
depth_t depth,
@@ -530,6 +551,15 @@ simdjson_warn_unused simdjson_inline simdjson_result<std::string_view> value_ite
simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iterator::get_raw_json_string() noexcept {
auto json = peek_scalar("string");
if (*json != '"') { return incorrect_type_error("Not a string"); }
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
if (_json_iter->allow_incomplete_json()) {
const size_t remaining_input_length = _json_iter->remaining_input_length(json);
const uint32_t max_len = remaining_input_length < peek_start_length() ? uint32_t(remaining_input_length) : peek_start_length();
if (!raw_json_string_is_quote_terminated(json, max_len)) {
return STRING_ERROR;
}
}
#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
advance_scalar("string");
return raw_json_string(json+1);
}
@@ -677,6 +707,15 @@ simdjson_warn_unused simdjson_inline simdjson_result<raw_json_string> value_iter
auto json = peek_scalar("string");
if (*json != '"') { return incorrect_type_error("Not a string"); }
if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; }
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
if (_json_iter->allow_incomplete_json()) {
const size_t remaining_input_length = _json_iter->remaining_input_length(json);
const uint32_t max_len = remaining_input_length < peek_root_length() ? uint32_t(remaining_input_length) : peek_root_length();
if (!raw_json_string_is_quote_terminated(json, max_len)) {
return STRING_ERROR;
}
}
#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
advance_scalar("string");
return raw_json_string(json+1);
}
@@ -1104,4 +1143,4 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value_iterato
} // namespace simdjson
#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H
#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H
tests/ondemand/ondemand_json_path_tests.cpp
+25
View File
@@ -243,6 +243,28 @@ namespace json_path_tests {
TEST_SUCCEED();
}
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
bool incomplete_string_value_with_allow_incomplete_json() {
TEST_START();
ondemand::parser parser;
ondemand::document doc;
std::string_view val;
auto incomplete_string_value = R"({"key":")"_padded;
auto complete_string_before_incomplete_value = R"({"key":"ok","incomplete":")"_padded;
ASSERT_SUCCESS(parser.iterate_allow_incomplete_json(incomplete_string_value).get(doc));
ASSERT_ERROR(doc.at_path(".key").get_string().get(val), simdjson::STRING_ERROR);
ASSERT_SUCCESS(parser.iterate_allow_incomplete_json(complete_string_before_incomplete_value).get(doc));
ASSERT_SUCCESS(doc.at_path(".key").get_string().get(val));
ASSERT_EQUAL(val, "ok");
ASSERT_SUCCESS(parser.iterate_allow_incomplete_json(complete_string_before_incomplete_value).get(doc));
ASSERT_ERROR(doc.at_path(".incomplete").get_string().get(val), simdjson::STRING_ERROR);
TEST_SUCCEED();
}
#endif
bool many_json_paths_object_array() {
TEST_START();
@@ -442,6 +464,9 @@ namespace json_path_tests {
many_json_paths_object_array() &&
many_json_paths_with_prefix() &&
run_broken_tests() &&
#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON
incomplete_string_value_with_allow_incomplete_json() &&
#endif
json_path_invalidation() &&
demo_test() &&
demo_relative_path() &&