From b3fe96cdb2776d16b7e49db06cf665106fc60583 Mon Sep 17 00:00:00 2001 From: jmestwa-coder Date: Wed, 29 Apr 2026 22:35:45 +0530 Subject: [PATCH] Align source() output for comma-delimited streams with json_sequence behavior (#2699) --- include/simdjson/dom/document_stream-inl.h | 2 +- .../generic/ondemand/document_stream-inl.h | 2 +- tests/dom/document_stream_tests.cpp | 21 +++++++----------- .../ondemand_document_stream_tests.cpp | 22 ++++++++----------- 4 files changed, 19 insertions(+), 28 deletions(-) diff --git a/include/simdjson/dom/document_stream-inl.h b/include/simdjson/dom/document_stream-inl.h index acdb07004..d43f55586 100644 --- a/include/simdjson/dom/document_stream-inl.h +++ b/include/simdjson/dom/document_stream-inl.h @@ -234,7 +234,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // cannot legally appear in a JSON value at the source level (control // characters in strings must be escaped as \u001E), so stripping it is // safe in every stream_format. - while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E)) { + while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) { svlen--; } return std::string_view(start, svlen); diff --git a/include/simdjson/generic/ondemand/document_stream-inl.h b/include/simdjson/generic/ondemand/document_stream-inl.h index 0300c0401..b63b198c4 100644 --- a/include/simdjson/generic/ondemand/document_stream-inl.h +++ b/include/simdjson/generic/ondemand/document_stream-inl.h @@ -398,7 +398,7 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc // legally appear in a JSON value at the source level (control // characters in strings must be escaped as \u001E), so stripping // it is safe in every stream_format. - while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E)) { + while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) { svlen--; } return std::string_view(start, svlen); diff --git a/tests/dom/document_stream_tests.cpp b/tests/dom/document_stream_tests.cpp index 5c6e2d6b1..5d2c08893 100644 --- a/tests/dom/document_stream_tests.cpp +++ b/tests/dom/document_stream_tests.cpp @@ -1771,41 +1771,36 @@ namespace document_stream_tests { // Note on source(): the comma_delimited filter strips root-level // commas from structural_indexes but the bytes remain in the buffer. // source() for a scalar slices [current_index, next_doc_index) and - // strips trailing whitespace, so it currently includes the trailing - // separator comma (e.g. doc 0 source = "1,"). This is the analogous - // bug to the json_sequence RS-trailing source bug. The assertions - // below lock down current_index() and parsed values strictly, and - // accept source() either with or without a single trailing comma. + // strips trailing whitespace; trailing delimiter is stripped for + // consistency with json_sequence. The assertions below lock down + // current_index() and parsed values strictly, and source() returns the + // JSON value without trailing delimiter. auto input = R"(1,2,"x",true,null)"_padded; ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream)); auto it = stream.begin(); - auto src_matches = [](std::string_view src, std::string_view want) { - return src == want || src == std::string(want) + ","; - }; - // doc 0: 1 ASSERT_EQUAL(it.current_index(), size_t(0)); int64_t i1; ASSERT_SUCCESS((*it).get(i1)); ASSERT_EQUAL(i1, int64_t(1)); - ASSERT_TRUE(src_matches(it.source(), "1")); + ASSERT_EQUAL(it.source(), std::string_view("1")); ++it; // doc 1: 2 ASSERT_EQUAL(it.current_index(), size_t(2)); int64_t i2; ASSERT_SUCCESS((*it).get(i2)); ASSERT_EQUAL(i2, int64_t(2)); - ASSERT_TRUE(src_matches(it.source(), "2")); + ASSERT_EQUAL(it.source(), std::string_view("2")); ++it; // doc 2: "x" ASSERT_EQUAL(it.current_index(), size_t(4)); std::string_view sv; ASSERT_SUCCESS((*it).get(sv)); ASSERT_EQUAL(sv, std::string_view("x")); - ASSERT_TRUE(src_matches(it.source(), "\"x\"")); + ASSERT_EQUAL(it.source(), std::string_view("\"x\"")); ++it; // doc 3: true ASSERT_EQUAL(it.current_index(), size_t(8)); bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b); - ASSERT_TRUE(src_matches(it.source(), "true")); + ASSERT_EQUAL(it.source(), std::string_view("true")); ++it; // doc 4: null (last doc, no trailing comma in source either way) diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp index 26eaf29ce..0ef22cc6d 100644 --- a/tests/ondemand/ondemand_document_stream_tests.cpp +++ b/tests/ondemand/ondemand_document_stream_tests.cpp @@ -1829,22 +1829,18 @@ namespace document_stream_tests { // Note on source(): the comma_delimited filter strips root-level // commas from structural_indexes but the bytes remain in the // buffer. source() for a scalar slices [current_index, - // next_doc_index) and strips trailing whitespace, so it - // currently includes the trailing separator comma (e.g. doc 0 - // source = "1,"). The assertions below lock down current_index() - // and parsed values strictly, and accept source() either with - // or without a single trailing comma. + // next_doc_index) and strips trailing whitespace; trailing + // delimiter is stripped for consistency with json_sequence. The + // assertions below lock down current_index() and parsed values + // strictly, and source() returns the JSON value without trailing + // delimiter. auto input = R"(1,2,"x",true,null)"_padded; ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream)); auto it = stream.begin(); - auto src_matches = [](std::string_view src, std::string_view want) { - return src == want || src == std::string(want) + ","; - }; - // doc 0: 1 ASSERT_EQUAL(it.current_index(), size_t(0)); - ASSERT_TRUE(src_matches(it.source(), "1")); + ASSERT_EQUAL(it.source(), std::string_view("1")); { ondemand::document_reference doc; ASSERT_SUCCESS((*it).get(doc)); @@ -1855,7 +1851,7 @@ namespace document_stream_tests { // doc 1: 2 ASSERT_EQUAL(it.current_index(), size_t(2)); - ASSERT_TRUE(src_matches(it.source(), "2")); + ASSERT_EQUAL(it.source(), std::string_view("2")); { ondemand::document_reference doc; ASSERT_SUCCESS((*it).get(doc)); @@ -1866,7 +1862,7 @@ namespace document_stream_tests { // doc 2: "x" ASSERT_EQUAL(it.current_index(), size_t(4)); - ASSERT_TRUE(src_matches(it.source(), "\"x\"")); + ASSERT_EQUAL(it.source(), std::string_view("\"x\"")); { ondemand::document_reference doc; ASSERT_SUCCESS((*it).get(doc)); @@ -1877,7 +1873,7 @@ namespace document_stream_tests { // doc 3: true ASSERT_EQUAL(it.current_index(), size_t(8)); - ASSERT_TRUE(src_matches(it.source(), "true")); + ASSERT_EQUAL(it.source(), std::string_view("true")); { ondemand::document_reference doc; ASSERT_SUCCESS((*it).get(doc));