admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Align source() output for comma-delimited streams with json_sequence behavior (#2699)

Browse Source
This commit is contained in:
jmestwa-coder
2026-04-29 22:35:45 +05:30
committed by GitHub
parent 45d296e219
commit b3fe96cdb2
4 changed files with 19 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/simdjson/dom/document_stream-inl.h
+1 -1
View File
@@ -234,7 +234,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
// cannot legally appear in a JSON value at the source level (control
// characters in strings must be escaped as \u001E), so stripping it is
// safe in every stream_format.
while(svlen > 1 && (std::isspace(static_cast<unsigned char>(start[svlen-1])) || start[svlen-1] == '\0' || static_cast<uint8_t>(start[svlen-1]) == 0x1E)) {
while(svlen > 1 && (std::isspace(static_cast<unsigned char>(start[svlen-1])) || start[svlen-1] == '\0' || static_cast<uint8_t>(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) {
svlen--;
}
return std::string_view(start, svlen);
include/simdjson/generic/ondemand/document_stream-inl.h
+1 -1
View File
@@ -398,7 +398,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
// legally appear in a JSON value at the source level (control
// characters in strings must be escaped as \u001E), so stripping
// it is safe in every stream_format.
while(svlen > 1 && (std::isspace(static_cast<unsigned char>(start[svlen-1])) || start[svlen-1] == '\0' || static_cast<uint8_t>(start[svlen-1]) == 0x1E)) {
while(svlen > 1 && (std::isspace(static_cast<unsigned char>(start[svlen-1])) || start[svlen-1] == '\0' || static_cast<uint8_t>(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) {
svlen--;
}
return std::string_view(start, svlen);
tests/dom/document_stream_tests.cpp
+8 -13
View File
@@ -1771,41 +1771,36 @@ namespace document_stream_tests {
// Note on source(): the comma_delimited filter strips root-level
// commas from structural_indexes but the bytes remain in the buffer.
// source() for a scalar slices [current_index, next_doc_index) and
// strips trailing whitespace, so it currently includes the trailing
// separator comma (e.g. doc 0 source = "1,"). This is the analogous
// bug to the json_sequence RS-trailing source bug. The assertions
// below lock down current_index() and parsed values strictly, and
// accept source() either with or without a single trailing comma.
// strips trailing whitespace; trailing delimiter is stripped for
// consistency with json_sequence. The assertions below lock down
// current_index() and parsed values strictly, and source() returns the
// JSON value without trailing delimiter.
auto input = R"(1,2,"x",true,null)"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
auto src_matches = [](std::string_view src, std::string_view want) {
return src == want || src == std::string(want) + ",";
};
// doc 0: 1
ASSERT_EQUAL(it.current_index(), size_t(0));
int64_t i1; ASSERT_SUCCESS((*it).get(i1)); ASSERT_EQUAL(i1, int64_t(1));
ASSERT_TRUE(src_matches(it.source(), "1"));
ASSERT_EQUAL(it.source(), std::string_view("1"));
++it;
// doc 1: 2
ASSERT_EQUAL(it.current_index(), size_t(2));
int64_t i2; ASSERT_SUCCESS((*it).get(i2)); ASSERT_EQUAL(i2, int64_t(2));
ASSERT_TRUE(src_matches(it.source(), "2"));
ASSERT_EQUAL(it.source(), std::string_view("2"));
++it;
// doc 2: "x"
ASSERT_EQUAL(it.current_index(), size_t(4));
std::string_view sv; ASSERT_SUCCESS((*it).get(sv)); ASSERT_EQUAL(sv, std::string_view("x"));
ASSERT_TRUE(src_matches(it.source(), "\"x\""));
ASSERT_EQUAL(it.source(), std::string_view("\"x\""));
++it;
// doc 3: true
ASSERT_EQUAL(it.current_index(), size_t(8));
bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b);
ASSERT_TRUE(src_matches(it.source(), "true"));
ASSERT_EQUAL(it.source(), std::string_view("true"));
++it;
// doc 4: null (last doc, no trailing comma in source either way)
tests/ondemand/ondemand_document_stream_tests.cpp
+9 -13
View File
@@ -1829,22 +1829,18 @@ namespace document_stream_tests {
// Note on source(): the comma_delimited filter strips root-level
// commas from structural_indexes but the bytes remain in the
// buffer. source() for a scalar slices [current_index,
// next_doc_index) and strips trailing whitespace, so it
// currently includes the trailing separator comma (e.g. doc 0
// source = "1,"). The assertions below lock down current_index()
// and parsed values strictly, and accept source() either with
// or without a single trailing comma.
// next_doc_index) and strips trailing whitespace; trailing
// delimiter is stripped for consistency with json_sequence. The
// assertions below lock down current_index() and parsed values
// strictly, and source() returns the JSON value without trailing
// delimiter.
auto input = R"(1,2,"x",true,null)"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
auto src_matches = [](std::string_view src, std::string_view want) {
return src == want || src == std::string(want) + ",";
};
// doc 0: 1
ASSERT_EQUAL(it.current_index(), size_t(0));
ASSERT_TRUE(src_matches(it.source(), "1"));
ASSERT_EQUAL(it.source(), std::string_view("1"));
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
@@ -1855,7 +1851,7 @@ namespace document_stream_tests {
// doc 1: 2
ASSERT_EQUAL(it.current_index(), size_t(2));
ASSERT_TRUE(src_matches(it.source(), "2"));
ASSERT_EQUAL(it.source(), std::string_view("2"));
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
@@ -1866,7 +1862,7 @@ namespace document_stream_tests {
// doc 2: "x"
ASSERT_EQUAL(it.current_index(), size_t(4));
ASSERT_TRUE(src_matches(it.source(), "\"x\""));
ASSERT_EQUAL(it.source(), std::string_view("\"x\""));
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
@@ -1877,7 +1873,7 @@ namespace document_stream_tests {
// doc 3: true
ASSERT_EQUAL(it.current_index(), size_t(8));
ASSERT_TRUE(src_matches(it.source(), "true"));
ASSERT_EQUAL(it.source(), std::string_view("true"));
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));