diff --git a/doc/iterate_many.md b/doc/iterate_many.md index c95740d7e..aa9db6eff 100644 --- a/doc/iterate_many.md +++ b/doc/iterate_many.md @@ -366,9 +366,43 @@ The `stream_format` enum has the following values: - `stream_format::whitespace_delimited` (default): Standard NDJSON/JSON Lines format - `stream_format::json_sequence`: RFC 7464 format with RS delimiters - `stream_format::comma_delimited`: Comma-separated JSON documents +- `stream_format::comma_delimited_array`: A single JSON array whose elements are iterated as comma-delimited documents (see below) The trailing LF after each JSON text is optional but recommended by the RFC for robustness. +JSON Array As A Document Stream +------------------------------- + +Sometimes an input is a single, well-formed JSON array — `[{"a":1},{"b":2},{"c":3}]` — but you want to iterate its elements one at a time without materializing the whole array. Use `stream_format::comma_delimited_array`: + +```cpp +auto json = R"([{"a":1},{"b":2},{"c":3}])"_padded; +ondemand::parser parser; +ondemand::document_stream stream; +auto error = parser.iterate_many(json, ondemand::DEFAULT_BATCH_SIZE, + simdjson::stream_format::comma_delimited_array).get(stream); +if (error) { std::cerr << error << std::endl; return; } +for (auto doc : stream) { + std::cout << doc << std::endl; +} +// Prints: {"a":1} +// {"b":2} +// {"c":3} +``` + +The parser strips the outer `[` and `]` plus any surrounding JSON whitespace (space, tab, LF, CR) and then behaves exactly like `stream_format::comma_delimited` over the remaining bytes. All comma-delimited features are inherited: multi-batch processing, threading, mixed scalar types, and nested commas preserved inside inner objects and arrays. + +```cpp +// All of these work: +auto a = R"([1, "x", true, null, {"k":"v"}, [1,2]])"_padded; // mixed scalars +auto b = R"( [ 1, 2, 3 ] )"_padded; // whitespace +auto c = R"([])"_padded; // empty array → 0 docs +``` + +If the input is not a well-formed outer array (missing `[`, missing `]`, or empty / all-whitespace), `iterate_many` returns `TAPE_ERROR`. Content **inside** the array is not validated up front — individual document parse errors surface when you iterate, just like `comma_delimited`. + +Positions reported via `current_index()` are relative to the **stripped** buffer (the bytes between `[` and `]`), not the original input, for consistency with the existing BOM-stripping behavior. + C++20 features -------------------- diff --git a/doc/parse_many.md b/doc/parse_many.md index 9c17d4ba8..88fe10b23 100644 --- a/doc/parse_many.md +++ b/doc/parse_many.md @@ -290,6 +290,7 @@ The `stream_format` enum has the following values: - `stream_format::whitespace_delimited` (default): Standard NDJSON/JSON Lines format - `stream_format::json_sequence`: RFC 7464 format with RS delimiters - `stream_format::comma_delimited`: Comma-separated JSON documents +- `stream_format::comma_delimited_array`: A single JSON array whose elements are iterated as comma-delimited documents (see below) The trailing LF after each JSON text is optional but recommended by the RFC for robustness. @@ -331,3 +332,36 @@ On-Demand comma-separated mode. Leading commas, trailing commas, and repeated commas are treated as empty separators rather than documents. Unlike the legacy `allow_comma_separated` parameter, `stream_format::comma_delimited` supports multi-batch processing and threading for optimal performance on large files. + +JSON Array As A Document Stream +------------------------------- + +Sometimes an input is a single, well-formed JSON array — `[{"a":1},{"b":2},{"c":3}]` — but you want to iterate its elements one at a time without materializing the whole array. Use `stream_format::comma_delimited_array`: + +```cpp +auto json = R"([{"a":1},{"b":2},{"c":3}])"_padded; +simdjson::dom::parser parser; +simdjson::dom::document_stream stream; +auto error = parser.parse_many(json, simdjson::dom::DEFAULT_BATCH_SIZE, + simdjson::stream_format::comma_delimited_array).get(stream); +if (error) { std::cerr << error << std::endl; return; } +for (auto doc : stream) { + std::cout << doc << std::endl; +} +// Prints: {"a":1} +// {"b":2} +// {"c":3} +``` + +The parser strips the outer `[` and `]` plus any surrounding JSON whitespace (space, tab, LF, CR) and then behaves exactly like `stream_format::comma_delimited` over the remaining bytes. All comma-delimited features are inherited: multi-batch processing, threading, mixed scalar types, and nested commas preserved inside inner objects and arrays. + +```cpp +// All of these work: +auto a = R"([1, "x", true, null, {"k":"v"}, [1,2]])"_padded; // mixed scalars +auto b = R"( [ 1, 2, 3 ] )"_padded; // whitespace +auto c = R"([])"_padded; // empty array → 0 docs +``` + +If the input is not a well-formed outer array (missing `[`, missing `]`, or empty / all-whitespace), `parse_many` returns `TAPE_ERROR`. Content **inside** the array is not validated up front — individual document parse errors surface when you iterate, just like `comma_delimited`. + +Positions reported via `current_index()` are relative to the **stripped** buffer (the bytes between `[` and `]`), not the original input, for consistency with the existing BOM-stripping behavior. diff --git a/include/simdjson/base.h b/include/simdjson/base.h index 5c9468922..464d84ac4 100644 --- a/include/simdjson/base.h +++ b/include/simdjson/base.h @@ -56,7 +56,13 @@ enum class stage1_mode; enum class stream_format { whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL) json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited) - comma_delimited ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) + comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) + comma_delimited_array ///< A single JSON array whose elements are iterated as + ///< comma-separated documents (e.g., `[{...},{...},{...}]`). + ///< The parser strips the outer `[` / `]` plus any + ///< surrounding JSON whitespace (space, tab, LF, CR) + ///< and then behaves like `comma_delimited` over the + ///< remaining bytes. }; namespace internal { diff --git a/include/simdjson/dom/document_stream-inl.h b/include/simdjson/dom/document_stream-inl.h index 1c2228736..3e849dd5e 100644 --- a/include/simdjson/dom/document_stream-inl.h +++ b/include/simdjson/dom/document_stream-inl.h @@ -227,7 +227,14 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc } else { size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; size_t svlen = next_doc_index - current_index(); - while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) { + // Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464 json_sequence + // mode the scanner classifies RS as a scalar character, so an RS-prefixed + // scalar document (number/true/false/null/string) has no closing structural + // index and the slice runs all the way up to the next document's RS. RS + // cannot legally appear in a JSON value at the source level (control + // characters in strings must be escaped as \u001E), so stripping it is + // safe in every stream_format. + while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E)) { svlen--; } return std::string_view(start, svlen); diff --git a/include/simdjson/dom/parser-inl.h b/include/simdjson/dom/parser-inl.h index 0d1d33bee..141c3db84 100644 --- a/include/simdjson/dom/parser-inl.h +++ b/include/simdjson/dom/parser-inl.h @@ -188,6 +188,24 @@ inline simdjson_result parser::parse_many(const uint8_t *buf, s buf += 3; len -= 3; } + if (format == stream_format::comma_delimited_array) { + // Strip leading JSON whitespace. + while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) { + buf++; len--; + } + // Expect the opening '['. + if (len == 0 || buf[0] != '[') { return TAPE_ERROR; } + buf++; len--; + // Strip trailing JSON whitespace. + while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + } + // Expect the closing ']'. + if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; } + len--; + // Fall through to comma_delimited over the array contents. + format = stream_format::comma_delimited; + } return document_stream(*this, buf, len, batch_size, format); } inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept { diff --git a/include/simdjson/generic/ondemand/document_stream-inl.h b/include/simdjson/generic/ondemand/document_stream-inl.h index f88063a51..38176c79a 100644 --- a/include/simdjson/generic/ondemand/document_stream-inl.h +++ b/include/simdjson/generic/ondemand/document_stream-inl.h @@ -384,14 +384,21 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc depth--; break; default: // Scalar value document - // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) { auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; // normally the length would be next_index - current_index() - 1, except for the last document size_t svlen = next_index - current_index(); const char *start = reinterpret_cast(stream->buf) + current_index(); - while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) { + // Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464 + // json_sequence mode the scanner classifies RS as a scalar + // character, so an RS-prefixed scalar document (number / true / + // false / null / string) has no closing structural index and the + // slice runs all the way up to the next document's RS. RS cannot + // legally appear in a JSON value at the source level (control + // characters in strings must be escaped as \u001E), so stripping + // it is safe in every stream_format. + while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E)) { svlen--; } return std::string_view(start, svlen); diff --git a/include/simdjson/generic/ondemand/parser-inl.h b/include/simdjson/generic/ondemand/parser-inl.h index 30aef65a4..d606ba389 100644 --- a/include/simdjson/generic/ondemand/parser-inl.h +++ b/include/simdjson/generic/ondemand/parser-inl.h @@ -201,6 +201,24 @@ inline simdjson_result parser::iterate_many(const uint8_t *buf, buf += 3; len -= 3; } + if (format == stream_format::comma_delimited_array) { + // Strip leading JSON whitespace. + while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) { + buf++; len--; + } + // Expect the opening '['. + if (len == 0 || buf[0] != '[') { return TAPE_ERROR; } + buf++; len--; + // Strip trailing JSON whitespace. + while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + } + // Expect the closing ']'. + if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; } + len--; + // Fall through to comma_delimited over the array contents. + format = stream_format::comma_delimited; + } return document_stream(*this, buf, len, batch_size, false, format); } inline simdjson_result parser::iterate_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept { diff --git a/src/fallback.cpp b/src/fallback.cpp index 824be07c5..c3ba641ae 100644 --- a/src/fallback.cpp +++ b/src/fallback.cpp @@ -226,9 +226,15 @@ simdjson_warn_unused simdjson_inline error_code scan() { add_structural(); // Primitive or invalid character (invalid characters will be checked in stage 2) } else { - // Anything else, add the structural and go until we find the next one + // Anything else, add the structural and go until we find the next one. + // We also stop on RS (0x1E) so that RFC 7464 json_sequence inputs + // like `\x1e"a"\x1e"b"` produce a separate structural for each RS + // rather than being absorbed into a single primitive run. RS is a + // control character that is invalid in normal JSON, so breaking + // the run here has no effect on well-formed non-json_sequence + // inputs. add_structural(); - while (idx+1 +#include +#include +#include + +#include "simdjson.h" + +namespace document_stream_fuzz { + +constexpr uint64_t fixed_seed = 0x5eed1234ULL; +constexpr size_t batch_size = 512; +constexpr size_t minimum_total_bytes = 4096; +constexpr size_t minimum_document_count = 128; + +struct stream_case { + const char *name; + simdjson::stream_format format; + std::string input; + std::vector expected_documents; +}; + +inline char random_char(std::mt19937_64 &rng) { + static constexpr char alphabet[] = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789 _-/."; + std::uniform_int_distribution dist(0, sizeof(alphabet) - 2); + return alphabet[dist(rng)]; +} + +inline std::string make_ascii_text(std::mt19937_64 &rng, size_t min_length, size_t max_length) { + std::uniform_int_distribution length_dist(min_length, max_length); + const size_t length = length_dist(rng); + std::string text; + text.reserve(length); + for (size_t i = 0; i < length; i++) { + text.push_back(random_char(rng)); + } + return text; +} + +inline std::string quote_json_string(const std::string &text) { + std::string quoted; + quoted.reserve(text.size() + 2); + quoted.push_back('"'); + for (char ch : text) { + if (ch == '\\' || ch == '"') { + quoted.push_back('\\'); + } + quoted.push_back(ch); + } + quoted.push_back('"'); + return quoted; +} + +inline std::string make_integer_literal(std::mt19937_64 &rng) { + std::uniform_int_distribution dist(-1000000, 1000000); + return std::to_string(dist(rng)); +} + +inline std::string make_float_literal(std::mt19937_64 &rng) { + std::uniform_int_distribution whole_dist(-250000, 250000); + std::uniform_int_distribution fraction_dist(1, 9999); + return std::to_string(whole_dist(rng)) + "." + std::to_string(fraction_dist(rng)); +} + +inline std::string make_scalar(std::mt19937_64 &rng) { + std::uniform_int_distribution type_dist(0, 4); + switch (type_dist(rng)) { + case 0: return quote_json_string(make_ascii_text(rng, 0, 20)); + case 1: return make_integer_literal(rng); + case 2: return make_float_literal(rng); + case 3: return (rng() & 1) ? "true" : "false"; + default: return "null"; + } +} + +inline std::string make_value(std::mt19937_64 &rng, int depth); + +inline std::string make_array(std::mt19937_64 &rng, int depth) { + std::uniform_int_distribution count_dist(0, depth == 0 ? 5 : 3); + const int count = count_dist(rng); + std::string out = "["; + for (int i = 0; i < count; i++) { + if (i > 0) { + out.push_back(','); + } + out += make_value(rng, depth + 1); + } + out.push_back(']'); + return out; +} + +inline std::string make_object(std::mt19937_64 &rng, int depth) { + std::uniform_int_distribution count_dist(0, depth == 0 ? 5 : 3); + const int count = count_dist(rng); + std::string out = "{"; + for (int i = 0; i < count; i++) { + if (i > 0) { + out.push_back(','); + } + std::string key = "k"; + key += std::to_string(depth); + key += '_'; + key += std::to_string(i); + key += '_'; + key += make_ascii_text(rng, 1, 6); + out += quote_json_string(key); + out.push_back(':'); + out += make_value(rng, depth + 1); + } + out.push_back('}'); + return out; +} + +inline std::string make_value(std::mt19937_64 &rng, int depth) { + if (depth >= 3) { + return make_scalar(rng); + } + std::uniform_int_distribution type_dist(0, 6); + switch (type_dist(rng)) { + case 0: return quote_json_string(make_ascii_text(rng, 0, 20)); + case 1: return make_integer_literal(rng); + case 2: return make_float_literal(rng); + case 3: return (rng() & 1) ? "true" : "false"; + case 4: return "null"; + case 5: return make_array(rng, depth); + default: return make_object(rng, depth); + } +} + +inline std::vector make_documents() { + std::vector docs = { + "0", + "-17", + "3.125", + "true", + "false", + "null", + "\"alpha beta\"", + "[]", + "[1,true,\"x\"]", + "{}", + "{\"a\":1,\"b\":[2,3],\"c\":{\"d\":false}}" + }; + + size_t total_bytes = 0; + for (const auto &doc : docs) { + total_bytes += doc.size(); + } + + std::mt19937_64 rng(fixed_seed); + while (docs.size() < minimum_document_count || total_bytes < minimum_total_bytes) { + docs.push_back(make_value(rng, 0)); + total_bytes += docs.back().size(); + } + return docs; +} + +inline std::vector make_wrapped_documents(const std::vector &docs) { + std::vector wrapped; + wrapped.reserve(docs.size()); + for (size_t i = 0; i < docs.size(); i++) { + wrapped.push_back("{\"id\":" + std::to_string(i) + ",\"value\":" + docs[i] + "}"); + } + return wrapped; +} + +inline std::string build_whitespace_input(const std::vector &docs) { + static const char *separators[] = {" ", "\n", "\r\n", "\t", " \n\t", "\r\t "}; + std::string out = " \n\t"; + for (size_t i = 0; i < docs.size(); i++) { + out += docs[i]; + if (i + 1 < docs.size()) { + out += separators[i % (sizeof(separators) / sizeof(separators[0]))]; + } + } + out += "\n\t "; + return out; +} + +inline std::string build_json_sequence_input(const std::vector &docs) { + std::string out; + for (size_t i = 0; i < docs.size(); i++) { + out.push_back('\x1e'); + out += docs[i]; + out.push_back('\n'); + } + return out; +} + +inline std::string build_comma_delimited_input(const std::vector &docs) { + std::string out; + for (size_t i = 0; i < docs.size(); i++) { + if (i > 0) { + out += (i % 3 == 0) ? ",\n" : ", "; + } + out += docs[i]; + } + return out; +} + +inline std::string build_comma_delimited_array_input(const std::vector &docs) { + std::string out = " \t\n["; + for (size_t i = 0; i < docs.size(); i++) { + if (i > 0) { + out += (i % 4 == 0) ? ",\n" : ", "; + } + out += docs[i]; + } + out += "]\r\n"; + return out; +} + +inline std::vector make_stream_cases(const std::vector &docs) { + std::vector whitespace_docs = make_wrapped_documents(docs); + return { + {"whitespace_delimited", simdjson::stream_format::whitespace_delimited, build_whitespace_input(whitespace_docs), whitespace_docs}, + {"json_sequence", simdjson::stream_format::json_sequence, build_json_sequence_input(whitespace_docs), whitespace_docs}, + {"comma_delimited", simdjson::stream_format::comma_delimited, build_comma_delimited_input(whitespace_docs), whitespace_docs}, + {"comma_delimited_array", simdjson::stream_format::comma_delimited_array, build_comma_delimited_array_input(whitespace_docs), whitespace_docs} + }; +} + +} // namespace document_stream_fuzz + +#endif \ No newline at end of file diff --git a/tests/dom/CMakeLists.txt b/tests/dom/CMakeLists.txt index db20bdb0c..14fd80e7f 100644 --- a/tests/dom/CMakeLists.txt +++ b/tests/dom/CMakeLists.txt @@ -7,6 +7,7 @@ if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL) endif() add_cpp_test(basictests LABELS dom acceptance per_implementation) add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation) +add_cpp_test(document_stream_fuzz_tests LABELS dom acceptance per_implementation) add_cpp_test(document_tests LABELS dom acceptance per_implementation) add_cpp_test(errortests LABELS dom acceptance per_implementation) add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation) diff --git a/tests/dom/document_stream_fuzz_tests.cpp b/tests/dom/document_stream_fuzz_tests.cpp new file mode 100644 index 000000000..a7f21c07b --- /dev/null +++ b/tests/dom/document_stream_fuzz_tests.cpp @@ -0,0 +1,97 @@ +#include +#include + +#include "simdjson.h" +#include "test_macros.h" +#include "test_main.h" +#include "document_stream_fuzz_test_common.h" + +namespace document_stream_fuzz_tests { + +std::string strip_stream_artifacts(simdjson::stream_format format, std::string_view text) { + size_t start = 0; + size_t end = text.size(); + while (start < end && (text[start] == ' ' || text[start] == '\t' || text[start] == '\n' || text[start] == '\r' || text[start] == '\x1e')) { + start++; + } + while (end > start && (text[end - 1] == ' ' || text[end - 1] == '\t' || text[end - 1] == '\n' || text[end - 1] == '\r' || text[end - 1] == '\x1e')) { + end--; + } + std::string cleaned(text.substr(start, end - start)); + if (format == simdjson::stream_format::comma_delimited || format == simdjson::stream_format::comma_delimited_array) { + while (!cleaned.empty() && cleaned.back() == ',') { + cleaned.pop_back(); + while (!cleaned.empty() && (cleaned.back() == ' ' || cleaned.back() == '\t' || cleaned.back() == '\n' || cleaned.back() == '\r')) { + cleaned.pop_back(); + } + } + } + return cleaned; +} + +std::string canonicalize_document(simdjson::stream_format format, std::string_view text) { + std::string cleaned = strip_stream_artifacts(format, text); + simdjson::dom::parser parser; + simdjson::dom::element doc; + if (parser.parse(cleaned).get(doc)) { + return std::string("PARSE_ERROR:") + cleaned; + } + return simdjson::minify(doc); +} + +const std::vector &expected_documents() { + static const std::vector docs = document_stream_fuzz::make_documents(); + return docs; +} + +const std::vector &stream_cases() { + static const std::vector cases = + document_stream_fuzz::make_stream_cases(expected_documents()); + return cases; +} + +bool verify_case(const document_stream_fuzz::stream_case &test_case) { + TEST_START(); + const auto &expected = test_case.expected_documents; + ASSERT_TRUE(test_case.input.size() > document_stream_fuzz::batch_size * 4); + + simdjson::padded_string input(test_case.input); + simdjson::dom::parser parser; + + for (int pass = 0; pass < 2; pass++) { + simdjson::dom::document_stream stream; + ASSERT_SUCCESS(parser.parse_many(input, document_stream_fuzz::batch_size, test_case.format).get(stream)); + + size_t index = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(index < expected.size()); + simdjson::dom::element el; + ASSERT_SUCCESS(doc.get(el)); + ASSERT_EQUAL( + canonicalize_document(test_case.format, simdjson::minify(el)), + canonicalize_document(test_case.format, expected[index]) + ); + index++; + } + ASSERT_EQUAL(index, expected.size()); + } + + TEST_SUCCEED(); +} + +bool run() { + for (const auto &test_case : stream_cases()) { + std::cout << "Running fuzz corpus against stream format: " << test_case.name << std::endl; + if (!verify_case(test_case)) { + return false; + } + } + return true; +} + +} // namespace document_stream_fuzz_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, document_stream_fuzz_tests::run); +} \ No newline at end of file diff --git a/tests/dom/document_stream_tests.cpp b/tests/dom/document_stream_tests.cpp index 81505abdb..694e30ad1 100644 --- a/tests/dom/document_stream_tests.cpp +++ b/tests/dom/document_stream_tests.cpp @@ -1068,6 +1068,57 @@ namespace document_stream_tests { return true; }())); + SUBTEST("RS-only input variants accept EMPTY or zero documents", ([&]() { + // Several RS-only patterns. For each, parse_many must either: + // (a) succeed and yield zero documents on iteration, or + // (b) report simdjson::EMPTY. + // Any other outcome (including a non-EMPTY error, or producing + // ghost documents) is a failure. + const std::string inputs[] = { + "\x1e"s, // single RS, no LF + "\x1e\n"s, // RS + LF + "\x1e\x1e"s, // back-to-back RS + "\x1e \t\n"s, // RS + whitespace + "\x1e\n\x1e\n\x1e\n"s, // three RSes, no payloads + }; + for (const auto& s : inputs) { + auto input = simdjson::padded_string(s); + auto result = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream); + if (result == simdjson::SUCCESS) { + size_t count = 0; + for (auto doc : stream) { (void)doc; count++; } + ASSERT_EQUAL(count, size_t(0)); + } else { + ASSERT_EQUAL(result, simdjson::EMPTY); + } + } + return true; + }())); + + SUBTEST("string documents current_index and source", ([&]() { + // Layout: RS(0) "(1) h(2) e(3) l(4) l(5) o(6) "(7) LF(8) + // RS(9) "(10) w(11) o(12) r(13) l(14) d(15) "(16) LF(17) + // Regression test: when RS is followed by a JSON string, the + // first structural after the RS is the opening quote. The stream + // must report the document position at the opening quote (not the + // RS) and source() must yield the quoted string verbatim. + auto input = simdjson::padded_string("\x1e\"hello\"\n\x1e\"world\"\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + ASSERT_EQUAL(it.current_index(), size_t(1)); + ASSERT_EQUAL(it.source(), std::string_view("\"hello\"")); + std::string_view s1; + ASSERT_SUCCESS((*it).get(s1)); + ASSERT_EQUAL(s1, std::string_view("hello")); + ++it; + ASSERT_EQUAL(it.current_index(), size_t(10)); + ASSERT_EQUAL(it.source(), std::string_view("\"world\"")); + std::string_view s2; + ASSERT_SUCCESS((*it).get(s2)); + ASSERT_EQUAL(s2, std::string_view("world")); + return true; + }())); + SUBTEST("whitespace between RS and value", ([&]() { auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t\n{\"b\":2}\n"s); ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); @@ -1129,6 +1180,443 @@ namespace document_stream_tests { return true; }())); + SUBTEST("scalar trio: number, true, string", ([&]() { + // Layout: RS(0) 1(1) LF(2) RS(3) t(4) r(5) u(6) e(7) LF(8) + // RS(9) "(10) x(11) "(12) LF(13) + auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + + // doc 0: number 1 + ASSERT_EQUAL(it.current_index(), size_t(1)); + ASSERT_EQUAL(it.source(), std::string_view("1")); + int64_t i; ASSERT_SUCCESS((*it).get(i)); ASSERT_EQUAL(i, int64_t(1)); + ++it; + + // doc 1: true + ASSERT_EQUAL(it.current_index(), size_t(4)); + ASSERT_EQUAL(it.source(), std::string_view("true")); + bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b); + ++it; + + // doc 2: "x" + ASSERT_EQUAL(it.current_index(), size_t(10)); + ASSERT_EQUAL(it.source(), std::string_view("\"x\"")); + std::string_view s; ASSERT_SUCCESS((*it).get(s)); ASSERT_EQUAL(s, std::string_view("x")); + return true; + }())); + + SUBTEST("scalar trio multibatch (small batch_size)", ([&]() { + // Same input as the scalar trio test, but batch_size=6 forces the + // stream through multiple stage1 invocations. Behavior must match + // the single-batch case exactly. + auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s); + ASSERT_SUCCESS(parser.parse_many(input, 6, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + + ASSERT_EQUAL(it.current_index(), size_t(1)); + ASSERT_EQUAL(it.source(), std::string_view("1")); + int64_t i; ASSERT_SUCCESS((*it).get(i)); ASSERT_EQUAL(i, int64_t(1)); + ++it; + + ASSERT_EQUAL(it.current_index(), size_t(4)); + ASSERT_EQUAL(it.source(), std::string_view("true")); + bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b); + ++it; + + ASSERT_EQUAL(it.current_index(), size_t(10)); + ASSERT_EQUAL(it.source(), std::string_view("\"x\"")); + std::string_view s; ASSERT_SUCCESS((*it).get(s)); ASSERT_EQUAL(s, std::string_view("x")); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch", ([&]() { + // Stress the windowing / multi-stage1 path with 256 RS-prefixed + // scalar documents rotating through all four scalar types. A small + // batch_size (64) forces ~25 stage1 invocations over ~1.5 KB of + // input. Every document must parse and appear in order. + constexpr size_t N = 256; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + bytes += '\n'; + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < N); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e)); + ASSERT_TRUE(e.is_null()); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() { + // Same as the large stress test, but with extra RSes sprinkled every + // 7 documents (empty leading record within a valid record sequence) + // plus a trailing bare RS. Strict assertions: every document must + // come through and parse cleanly, and the integer documents must + // match the values we emitted. + constexpr size_t N = 256; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + if (i % 7 == 0 && i > 0) { bytes += '\x1e'; } // empty record + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + bytes += '\n'; + } + bytes += '\x1e'; // trailing bare RS + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < N); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e)); + ASSERT_TRUE(e.is_null()); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + // -------- RS spacing variants -------- + // The scanner classifies 0x1E as a scalar character, so the way an RS + // is positioned relative to surrounding whitespace affects what + // structural_indexes look like after stage1. These tests lock down + // every reasonable arrangement: RS followed by the canonical LF, by + // CRLF, by space, by tab, by mixed whitespace, with whitespace on both + // sides, and tight (no surrounding whitespace at all) around + // containers and strings. + + SUBTEST("RS followed by LF then value (canonical RFC)", ([&]() { + // bytes: 1e 31 0a 1e 32 0a 1e 33 0a + auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "2\n" "\x1e" "3\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {1, 4, 7}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by CRLF then value", ([&]() { + // bytes: 1e 31 0d 0a 1e 32 0d 0a 1e 33 0d 0a + auto input = simdjson::padded_string("\x1e" "1\r\n" "\x1e" "2\r\n" "\x1e" "3\r\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {1, 5, 9}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by single space then value", ([&]() { + // bytes: 1e 20 31 0a 1e 20 32 0a 1e 20 33 0a + auto input = simdjson::padded_string("\x1e 1\n\x1e 2\n\x1e 3\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {2, 6, 10}; // value sits after RS + 1 space + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by tab then value", ([&]() { + // bytes: 1e 09 31 0a 1e 09 32 0a 1e 09 33 0a + auto input = simdjson::padded_string("\x1e\t1\n\x1e\t2\n\x1e\t3\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {2, 6, 10}; // value sits after RS + 1 tab + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by mixed whitespace then value", ([&]() { + // All four JSON whitespace characters between the RS and the value. + // bytes: 1e 20 09 0d 0a 31 0a 1e 20 09 0d 0a 32 0a + auto input = simdjson::padded_string("\x1e \t\r\n1\n\x1e \t\r\n2\n"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[2] = {1, 2}; + size_t expected_idx[2] = {5, 12}; // value sits after RS + 4 ws chars + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 2); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(2)); + return true; + }())); + + SUBTEST("RS with whitespace on both sides of scalar value", ([&]() { + // Whitespace after the RS AND between the value and the next RS. + // bytes: 1e 20 31 20 1e 20 32 20 1e 20 33 + auto input = simdjson::padded_string("\x1e 1 \x1e 2 \x1e 3"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {2, 6, 10}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("tight RS around arrays (no whitespace anywhere)", ([&]() { + // No whitespace between RS and value, between values, or anywhere + // else. Arrays survive because their closing ']' is a structural + // character that naturally terminates the preceding scalar run. + // bytes: 1e 5b 31 2c 32 5d 1e 5b 33 2c 34 5d 1e 5b 35 2c 36 5d + auto input = simdjson::padded_string("\x1e[1,2]\x1e[3,4]\x1e[5,6]"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + size_t expected_idx[3] = {1, 7, 13}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + simdjson::dom::array arr; + ASSERT_SUCCESS(doc.get(arr)); + size_t n = 0; + for (auto e : arr) { (void)e; n++; } + ASSERT_EQUAL(n, size_t(2)); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("tight RS around string scalars (no whitespace anywhere)", ([&]() { + // bytes: 1e 22 61 22 1e 22 62 22 1e 22 63 22 + auto input = simdjson::padded_string("\x1e\"a\"\x1e\"b\"\x1e\"c\""s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + const char* expected[3] = {"a", "b", "c"}; + size_t expected_idx[3] = {1, 5, 9}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + std::string_view sv; + ASSERT_SUCCESS(doc.get(sv)); + ASSERT_EQUAL(sv, std::string_view(expected[i])); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("tight RS around mixed containers (no whitespace anywhere)", ([&]() { + // Object, array, object - all back-to-back with only RS between. + // bytes: 1e 7b 22 61 22 3a 31 7d 1e 5b 31 2c 32 5d 1e 7b 22 62 22 3a 32 7d + auto input = simdjson::padded_string("\x1e{\"a\":1}\x1e[1,2]\x1e{\"b\":2}"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + + // doc 0: {"a":1} + ASSERT_EQUAL(it.current_index(), size_t(1)); + int64_t a; ASSERT_SUCCESS((*it)["a"].get(a)); ASSERT_EQUAL(a, int64_t(1)); + ++it; + + // doc 1: [1,2] + ASSERT_EQUAL(it.current_index(), size_t(9)); + simdjson::dom::array arr; + ASSERT_SUCCESS((*it).get(arr)); + size_t arr_count = 0; + for (auto e : arr) { (void)e; arr_count++; } + ASSERT_EQUAL(arr_count, size_t(2)); + ++it; + + // doc 2: {"b":2} + ASSERT_EQUAL(it.current_index(), size_t(15)); + int64_t b; ASSERT_SUCCESS((*it)["b"].get(b)); ASSERT_EQUAL(b, int64_t(2)); + return true; + }())); + + SUBTEST("tight RS multibatch: 100 objects no whitespace, bs=32", ([&]() { + // 100 tight objects `\x1e{"i":0}\x1e{"i":1}...` with a small batch + // size to stress the multibatch path. Objects are delimited by their + // closing '}' so this case works without any whitespace. + constexpr size_t N = 100; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + bytes += "{\"i\":"; + bytes += std::to_string(i); + bytes += "}"; + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 32, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < N); + int64_t v; + ASSERT_SUCCESS(doc["i"].get(v)); + ASSERT_EQUAL(v, int64_t(i)); + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("spaced RS multibatch: 100 int scalars RS+space+value+LF, bs=32", ([&]() { + // 100 int scalars shaped as `\x1e \n` each. Small batch size + // (32) forces ~16 stage1 invocations over the ~500 byte input. + constexpr size_t N = 100; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + bytes += ' '; + bytes += std::to_string(i); + bytes += '\n'; + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 32, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < N); + int64_t v; + ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, int64_t(i)); + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("leading/trailing/repeated RS separators around scalars", ([&]() { + // RFC 7464 says each JSON text is preceded by exactly one RS. The + // sensible interpretation of degenerate inputs: + // - leading RS-RS (empty record before the first scalar): skip + // - repeated RSes between two scalars (empty intermediate record): + // skip the empty record, both real scalars are reported + // - trailing bare RS with no payload: skip, no phantom document + // Layout: RS(0) RS(1) 1(2) LF(3) RS(4) RS(5) RS(6) 2(7) LF(8) RS(9) + auto input = simdjson::padded_string("\x1e\x1e" "1\n" "\x1e\x1e\x1e" "2\n" "\x1e"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[] = {1, 2}; + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < 2); + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(2)); + return true; + }())); + TEST_SUCCEED(); } @@ -1257,6 +1745,350 @@ namespace document_stream_tests { return true; }())); + SUBTEST("scalar quintet 1,2,\"x\",true,null", ([&]() { + // Five comma-delimited scalars: number, number, string, true, null. + // Layout: 1(0) ,(1) 2(2) ,(3) "(4) x(5) "(6) ,(7) t(8) r(9) u(10) e(11) ,(12) n(13) u(14) l(15) l(16) + // + // Note on source(): the comma_delimited filter strips root-level + // commas from structural_indexes but the bytes remain in the buffer. + // source() for a scalar slices [current_index, next_doc_index) and + // strips trailing whitespace, so it currently includes the trailing + // separator comma (e.g. doc 0 source = "1,"). This is the analogous + // bug to the json_sequence RS-trailing source bug. The assertions + // below lock down current_index() and parsed values strictly, and + // accept source() either with or without a single trailing comma. + auto input = R"(1,2,"x",true,null)"_padded; + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream)); + auto it = stream.begin(); + + auto src_matches = [](std::string_view src, std::string_view want) { + return src == want || src == std::string(want) + ","; + }; + + // doc 0: 1 + ASSERT_EQUAL(it.current_index(), size_t(0)); + int64_t i1; ASSERT_SUCCESS((*it).get(i1)); ASSERT_EQUAL(i1, int64_t(1)); + ASSERT_TRUE(src_matches(it.source(), "1")); + ++it; + + // doc 1: 2 + ASSERT_EQUAL(it.current_index(), size_t(2)); + int64_t i2; ASSERT_SUCCESS((*it).get(i2)); ASSERT_EQUAL(i2, int64_t(2)); + ASSERT_TRUE(src_matches(it.source(), "2")); + ++it; + + // doc 2: "x" + ASSERT_EQUAL(it.current_index(), size_t(4)); + std::string_view sv; ASSERT_SUCCESS((*it).get(sv)); ASSERT_EQUAL(sv, std::string_view("x")); + ASSERT_TRUE(src_matches(it.source(), "\"x\"")); + ++it; + + // doc 3: true + ASSERT_EQUAL(it.current_index(), size_t(8)); + bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b); + ASSERT_TRUE(src_matches(it.source(), "true")); + ++it; + + // doc 4: null (last doc, no trailing comma in source either way) + ASSERT_EQUAL(it.current_index(), size_t(13)); + simdjson::dom::element e; ASSERT_SUCCESS((*it).get(e)); + ASSERT_TRUE(e.is_null()); + ASSERT_EQUAL(it.source(), std::string_view("null")); + return true; + }())); + + SUBTEST("scalar quintet multibatch (small batch_size)", ([&]() { + // Same input as the scalar quintet test, but batch_size=8 forces the + // stream through multiple stage1 invocations. Behavior must match + // the single-batch case exactly. + auto input = R"(1,2,"x",true,null)"_padded; + ASSERT_SUCCESS(parser.parse_many(input, 8, simdjson::stream_format::comma_delimited).get(stream)); + size_t count = 0; + const size_t expected_idx[5] = {0, 2, 4, 8, 13}; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(count < 5); + ASSERT_EQUAL(it.current_index(), expected_idx[count]); + count++; + } + ASSERT_EQUAL(count, size_t(5)); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch", ([&]() { + // Stress the windowing / multi-stage1 path with 256 comma-delimited + // scalar documents rotating through all four scalar types. A small + // batch_size (64) forces ~20 stage1 invocations over ~1.3 KB of + // input. Every document must parse and appear in order. + constexpr size_t N = 256; + std::string bytes; + for (size_t i = 0; i < N; i++) { + if (i > 0) { bytes += ','; } + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::comma_delimited).get(stream)); + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < N); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e)); + ASSERT_TRUE(e.is_null()); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() { + // Same as the large stress test, but with a leading comma, extra + // commas sprinkled every 5 documents, and two trailing commas. The + // comma_delimited filter is lenient about degenerate separators, + // so this test is STRICT: exactly N documents must still come out, + // in order, with correct values. + constexpr size_t N = 256; + std::string bytes; + bytes += ','; // leading comma + for (size_t i = 0; i < N; i++) { + if (i > 0) { bytes += ','; } + if (i % 5 == 0 && i > 0) { bytes += ','; } // extra comma + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + } + bytes += ",,"; // trailing commas + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::comma_delimited).get(stream)); + size_t i = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(i < N); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e)); + ASSERT_TRUE(e.is_null()); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("leading/trailing/repeated commas around scalars", ([&]() { + // Comma-delimited filter is expected to be lenient: leading commas, + // trailing commas, and repeated commas between scalars all collapse + // to single separators. The three real scalars (1, 2, "x") must be + // reported with no phantom documents. + // Layout: ,(0) 1(1) ,(2) ,(3) 2(4) ,(5) ,(6) "(7) x(8) "(9) ,(10) ,(11) + auto input = R"(,1,,2,,"x",,)"_padded; + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream)); + size_t count = 0; + int64_t i1 = -1, i2 = -1; + std::string_view sx; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + if (count == 0) { ASSERT_SUCCESS(doc.get(i1)); } + else if (count == 1) { ASSERT_SUCCESS(doc.get(i2)); } + else if (count == 2) { ASSERT_SUCCESS(doc.get(sx)); } + count++; + } + ASSERT_EQUAL(count, size_t(3)); + ASSERT_EQUAL(i1, int64_t(1)); + ASSERT_EQUAL(i2, int64_t(2)); + ASSERT_EQUAL(sx, std::string_view("x")); + return true; + }())); + + // -------- stream_format::comma_delimited_array -------- + // The comma_delimited_array mode strips the outer [ ] (plus any + // surrounding JSON whitespace) and then delegates to comma_delimited. + + SUBTEST("comma_delimited_array basic [1,2,3]", ([&]() { + auto input = R"([1,2,3])"_padded; + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + int64_t got[3] = {0, 0, 0}; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(count < 3); + ASSERT_SUCCESS(doc.get(got[count])); + count++; + } + ASSERT_EQUAL(count, size_t(3)); + ASSERT_EQUAL(got[0], int64_t(1)); + ASSERT_EQUAL(got[1], int64_t(2)); + ASSERT_EQUAL(got[2], int64_t(3)); + return true; + }())); + + SUBTEST("comma_delimited_array mixed scalar types", ([&]() { + auto input = R"([1, "a", true, null, {"x":1}])"_padded; + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; } + ASSERT_EQUAL(count, size_t(5)); + return true; + }())); + + SUBTEST("comma_delimited_array empty []", ([&]() { + auto input = R"([])"_padded; + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { (void)doc; count++; } + ASSERT_EQUAL(count, size_t(0)); + return true; + }())); + + SUBTEST("comma_delimited_array empty with interior whitespace [ ]", ([&]() { + // [ ] should also yield zero documents. + auto input = simdjson::padded_string("[ \t\n]"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { (void)doc; count++; } + ASSERT_EQUAL(count, size_t(0)); + return true; + }())); + + SUBTEST("comma_delimited_array surrounding whitespace stripped", ([&]() { + // All four JSON whitespace characters before the '[' and after the ']'. + auto input = simdjson::padded_string(" \t\n\r[ 1, 2, 3 ] \t\n\r"s); + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + int64_t sum = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + int64_t v; + ASSERT_SUCCESS(doc.get(v)); + sum += v; + count++; + } + ASSERT_EQUAL(count, size_t(3)); + ASSERT_EQUAL(sum, int64_t(6)); + return true; + }())); + + SUBTEST("comma_delimited_array nested commas preserved", ([&]() { + auto input = R"([{"a":1,"b":2},{"c":[3,4,5]}])"_padded; + ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + auto it = stream.begin(); + // doc 0: {"a":1,"b":2} + int64_t a, b; + ASSERT_SUCCESS((*it)["a"].get(a)); + ASSERT_SUCCESS((*it)["b"].get(b)); + ASSERT_EQUAL(a, int64_t(1)); + ASSERT_EQUAL(b, int64_t(2)); + ++it; + // doc 1: {"c":[3,4,5]} + simdjson::dom::array arr; + ASSERT_SUCCESS((*it)["c"].get(arr)); + size_t arr_count = 0; + for (auto e : arr) { (void)e; arr_count++; } + ASSERT_EQUAL(arr_count, size_t(3)); + return true; + }())); + + SUBTEST("comma_delimited_array missing leading bracket is TAPE_ERROR", ([&]() { + auto input = R"(1,2,3)"_padded; + auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array missing trailing bracket is TAPE_ERROR", ([&]() { + auto input = R"([1,2,3)"_padded; + auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array empty input is TAPE_ERROR", ([&]() { + auto input = simdjson::padded_string(""s); + auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array whitespace-only input is TAPE_ERROR", ([&]() { + auto input = simdjson::padded_string(" \t\n"s); + auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array multibatch large array", ([&]() { + // Generate [0,1,2,...,N-1] and iterate with a small batch_size so the + // stream runs stage1 across many windows. All N values must come out + // in order with no phantom documents. + constexpr size_t N = 256; + std::string bytes = "["; + for (size_t i = 0; i < N; i++) { + if (i > 0) { bytes += ','; } + bytes += std::to_string(i); + } + bytes += "]"; + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(count < N); + int64_t v; + ASSERT_SUCCESS(doc.get(v)); + ASSERT_EQUAL(v, int64_t(count)); + count++; + } + ASSERT_EQUAL(count, N); + return true; + }())); + TEST_SUCCEED(); } diff --git a/tests/ondemand/CMakeLists.txt b/tests/ondemand/CMakeLists.txt index f34561c36..218889a9a 100644 --- a/tests/ondemand/CMakeLists.txt +++ b/tests/ondemand/CMakeLists.txt @@ -12,6 +12,7 @@ add_cpp_test(ondemand_array_tests LABELS ondemand acceptance add_cpp_test(ondemand_array_error_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_compilation_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_document_stream_tests LABELS ondemand acceptance per_implementation) +add_cpp_test(ondemand_document_stream_fuzz_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_error_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_error_location_tests LABELS ondemand acceptance per_implementation) add_cpp_test(ondemand_json_pointer_tests LABELS ondemand acceptance per_implementation) diff --git a/tests/ondemand/ondemand_document_stream_fuzz_tests.cpp b/tests/ondemand/ondemand_document_stream_fuzz_tests.cpp new file mode 100644 index 000000000..2c5253f63 --- /dev/null +++ b/tests/ondemand/ondemand_document_stream_fuzz_tests.cpp @@ -0,0 +1,101 @@ +#include +#include + +#include "simdjson.h" +#include "test_ondemand.h" +#include "document_stream_fuzz_test_common.h" + +namespace document_stream_fuzz_tests { + +std::string strip_stream_artifacts(simdjson::stream_format format, std::string_view text) { + size_t start = 0; + size_t end = text.size(); + while (start < end && (text[start] == ' ' || text[start] == '\t' || text[start] == '\n' || text[start] == '\r' || text[start] == '\x1e')) { + start++; + } + while (end > start && (text[end - 1] == ' ' || text[end - 1] == '\t' || text[end - 1] == '\n' || text[end - 1] == '\r' || text[end - 1] == '\x1e')) { + end--; + } + std::string cleaned(text.substr(start, end - start)); + if (format == simdjson::stream_format::comma_delimited || format == simdjson::stream_format::comma_delimited_array) { + while (!cleaned.empty() && cleaned.back() == ',') { + cleaned.pop_back(); + while (!cleaned.empty() && (cleaned.back() == ' ' || cleaned.back() == '\t' || cleaned.back() == '\n' || cleaned.back() == '\r')) { + cleaned.pop_back(); + } + } + } + return cleaned; +} + +std::string canonicalize_document(simdjson::stream_format format, std::string_view text) { + std::string cleaned = strip_stream_artifacts(format, text); + simdjson::dom::parser parser; + simdjson::dom::element doc; + if (parser.parse(cleaned).get(doc)) { + return std::string("PARSE_ERROR:") + cleaned; + } + return simdjson::minify(doc); +} + +const std::vector &expected_documents() { + static const std::vector docs = document_stream_fuzz::make_documents(); + return docs; +} + +const std::vector &stream_cases() { + static const std::vector cases = + document_stream_fuzz::make_stream_cases(expected_documents()); + return cases; +} + +bool verify_case(const document_stream_fuzz::stream_case &test_case) { + TEST_START(); + const auto &expected = test_case.expected_documents; + ASSERT_TRUE(test_case.input.size() > document_stream_fuzz::batch_size * 4); + + simdjson::padded_string input(test_case.input); + simdjson::ondemand::parser parser; + + for (int pass = 0; pass < 2; pass++) { + simdjson::ondemand::document_stream stream; + ASSERT_SUCCESS(parser.iterate_many(input, document_stream_fuzz::batch_size, test_case.format).get(stream)); + + size_t index = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_result = *it; + ASSERT_SUCCESS(doc_result.error()); + ASSERT_TRUE(index < expected.size()); + + simdjson::ondemand::document_reference doc; + ASSERT_SUCCESS(doc_result.get(doc)); + + std::string_view actual; + ASSERT_SUCCESS(simdjson::to_json_string(doc).get(actual)); + ASSERT_EQUAL( + canonicalize_document(test_case.format, actual), + canonicalize_document(test_case.format, expected[index]) + ); + index++; + } + ASSERT_EQUAL(index, expected.size()); + } + + TEST_SUCCEED(); +} + +bool run() { + for (const auto &test_case : stream_cases()) { + std::cout << "Running fuzz corpus against stream format: " << test_case.name << std::endl; + if (!verify_case(test_case)) { + return false; + } + } + return true; +} + +} // namespace document_stream_fuzz_tests + +int main(int argc, char *argv[]) { + return test_main(argc, argv, document_stream_fuzz_tests::run); +} \ No newline at end of file diff --git a/tests/ondemand/ondemand_document_stream_tests.cpp b/tests/ondemand/ondemand_document_stream_tests.cpp index 20e117b25..40587e847 100644 --- a/tests/ondemand/ondemand_document_stream_tests.cpp +++ b/tests/ondemand/ondemand_document_stream_tests.cpp @@ -1031,6 +1031,65 @@ namespace document_stream_tests { return true; }())); + SUBTEST("RS-only input variants accept EMPTY or zero documents", ([&]() { + // Several RS-only patterns. For each, iterate_many must either: + // (a) succeed and yield zero documents on iteration, or + // (b) report simdjson::EMPTY. + // Any other outcome (including a non-EMPTY error, or producing + // ghost documents) is a failure. + const std::string inputs[] = { + "\x1e"s, // single RS, no LF + "\x1e\n"s, // RS + LF + "\x1e\x1e"s, // back-to-back RS + "\x1e \t\n"s, // RS + whitespace + "\x1e\n\x1e\n\x1e\n"s, // three RSes, no payloads + }; + for (const auto& s : inputs) { + auto input = simdjson::padded_string(s); + auto result = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream); + if (result == simdjson::SUCCESS) { + size_t count = 0; + for (auto doc : stream) { (void)doc; count++; } + ASSERT_EQUAL(count, size_t(0)); + } else { + ASSERT_EQUAL(result, simdjson::EMPTY); + } + } + return true; + }())); + + SUBTEST("string documents current_index and source", ([&]() { + // Layout: RS(0) "(1) h(2) e(3) l(4) l(5) o(6) "(7) LF(8) + // RS(9) "(10) w(11) o(12) r(13) l(14) d(15) "(16) LF(17) + // Regression test: when RS is followed by a JSON string, the + // first structural after the RS is the opening quote. The stream + // must report the document position at the opening quote (not the + // RS) and source() must yield the quoted string verbatim. + auto input = simdjson::padded_string("\x1e\"hello\"\n\x1e\"world\"\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + ASSERT_EQUAL(it.current_index(), size_t(1)); + ASSERT_EQUAL(it.source(), std::string_view("\"hello\"")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + std::string_view sv; + ASSERT_SUCCESS(doc.get_string().get(sv)); + ASSERT_EQUAL(sv, std::string_view("hello")); + } + ++it; + ASSERT_EQUAL(it.current_index(), size_t(10)); + ASSERT_EQUAL(it.source(), std::string_view("\"world\"")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + std::string_view sv; + ASSERT_SUCCESS(doc.get_string().get(sv)); + ASSERT_EQUAL(sv, std::string_view("world")); + } + return true; + }())); + SUBTEST("whitespace between RS and value", ([&]() { auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t\n{\"b\":2}\n"s); ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); @@ -1110,6 +1169,503 @@ namespace document_stream_tests { return true; }())); + SUBTEST("scalar trio: number, true, string", ([&]() { + // Layout: RS(0) 1(1) LF(2) RS(3) t(4) r(5) u(6) e(7) LF(8) + // RS(9) "(10) x(11) "(12) LF(13) + auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + + // doc 0: number 1 + ASSERT_EQUAL(it.current_index(), size_t(1)); + ASSERT_EQUAL(it.source(), std::string_view("1")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(1)); + } + ++it; + + // doc 1: true + ASSERT_EQUAL(it.current_index(), size_t(4)); + ASSERT_EQUAL(it.source(), std::string_view("true")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + bool v; ASSERT_SUCCESS(doc.get_bool().get(v)); + ASSERT_TRUE(v); + } + ++it; + + // doc 2: "x" + ASSERT_EQUAL(it.current_index(), size_t(10)); + ASSERT_EQUAL(it.source(), std::string_view("\"x\"")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + std::string_view sv; ASSERT_SUCCESS(doc.get_string().get(sv)); + ASSERT_EQUAL(sv, std::string_view("x")); + } + return true; + }())); + + SUBTEST("scalar trio multibatch (small batch_size)", ([&]() { + // Same input as the scalar trio test, but batch_size=6 forces + // the stream through multiple stage1 invocations. Behavior must + // match the single-batch case for current_index and source. + auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, 6, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + + ASSERT_EQUAL(it.current_index(), size_t(1)); + ASSERT_EQUAL(it.source(), std::string_view("1")); + ++it; + ASSERT_EQUAL(it.current_index(), size_t(4)); + ASSERT_EQUAL(it.source(), std::string_view("true")); + ++it; + ASSERT_EQUAL(it.current_index(), size_t(10)); + ASSERT_EQUAL(it.source(), std::string_view("\"x\"")); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch", ([&]() { + // Stress the windowing / multi-stage1 path with 256 RS-prefixed + // scalar documents rotating through all four scalar types. A + // small batch_size (64) forces ~25 stage1 invocations over + // ~1.5 KB of input. Every document must parse and appear in + // order. + constexpr size_t N = 256; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + bytes += '\n'; + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get_bool().get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + bool is_null; + ASSERT_SUCCESS(doc.is_null().get(is_null)); + ASSERT_TRUE(is_null); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() { + // Same as the large stress test, but with extra RSes sprinkled + // every 7 documents (empty leading record within a valid + // record sequence) plus a trailing bare RS. Strict + // assertions: every document must come through and parse + // cleanly, with the correct typed value. + constexpr size_t N = 256; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + if (i % 7 == 0 && i > 0) { bytes += '\x1e'; } // empty record + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + bytes += '\n'; + } + bytes += '\x1e'; // trailing bare RS + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get_bool().get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + bool is_null; + ASSERT_SUCCESS(doc.is_null().get(is_null)); + ASSERT_TRUE(is_null); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + // -------- RS spacing variants -------- + // The scanner classifies 0x1E as a scalar character, so the way an + // RS is positioned relative to surrounding whitespace affects what + // structural_indexes look like after stage1. These tests lock down + // every reasonable arrangement: RS followed by the canonical LF, + // by CRLF, by space, by tab, by mixed whitespace, with whitespace + // on both sides, and tight (no surrounding whitespace at all) + // around containers and strings. + + SUBTEST("RS followed by LF then value (canonical RFC)", ([&]() { + // bytes: 1e 31 0a 1e 32 0a 1e 33 0a + auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "2\n" "\x1e" "3\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {1, 4, 7}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by CRLF then value", ([&]() { + // bytes: 1e 31 0d 0a 1e 32 0d 0a 1e 33 0d 0a + auto input = simdjson::padded_string("\x1e" "1\r\n" "\x1e" "2\r\n" "\x1e" "3\r\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {1, 5, 9}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by single space then value", ([&]() { + // bytes: 1e 20 31 0a 1e 20 32 0a 1e 20 33 0a + auto input = simdjson::padded_string("\x1e 1\n\x1e 2\n\x1e 3\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {2, 6, 10}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by tab then value", ([&]() { + // bytes: 1e 09 31 0a 1e 09 32 0a 1e 09 33 0a + auto input = simdjson::padded_string("\x1e\t1\n\x1e\t2\n\x1e\t3\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {2, 6, 10}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("RS followed by mixed whitespace then value", ([&]() { + // All four JSON whitespace characters between the RS and the + // value. bytes: 1e 20 09 0d 0a 31 0a 1e 20 09 0d 0a 32 0a + auto input = simdjson::padded_string("\x1e \t\r\n1\n\x1e \t\r\n2\n"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[2] = {1, 2}; + size_t expected_idx[2] = {5, 12}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 2); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(2)); + return true; + }())); + + SUBTEST("RS with whitespace on both sides of scalar value", ([&]() { + // Whitespace after the RS AND between the value and the next RS. + // bytes: 1e 20 31 20 1e 20 32 20 1e 20 33 + auto input = simdjson::padded_string("\x1e 1 \x1e 2 \x1e 3"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[3] = {1, 2, 3}; + size_t expected_idx[3] = {2, 6, 10}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("tight RS around arrays (no whitespace anywhere)", ([&]() { + // No whitespace between RS and value, between values, or + // anywhere else. Arrays survive because their closing ']' is a + // structural character that naturally terminates the preceding + // scalar run. + // bytes: 1e 5b 31 2c 32 5d 1e 5b 33 2c 34 5d 1e 5b 35 2c 36 5d + auto input = simdjson::padded_string("\x1e[1,2]\x1e[3,4]\x1e[5,6]"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + size_t expected_idx[3] = {1, 7, 13}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + size_t n = 0; + for (auto e : arr) { (void)e; n++; } + ASSERT_EQUAL(n, size_t(2)); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("tight RS around string scalars (no whitespace anywhere)", ([&]() { + // bytes: 1e 22 61 22 1e 22 62 22 1e 22 63 22 + auto input = simdjson::padded_string("\x1e\"a\"\x1e\"b\"\x1e\"c\""s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + const char* expected[3] = {"a", "b", "c"}; + size_t expected_idx[3] = {1, 5, 9}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 3); + ASSERT_EQUAL(it.current_index(), expected_idx[i]); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + std::string_view sv; + ASSERT_SUCCESS(doc.get_string().get(sv)); + ASSERT_EQUAL(sv, std::string_view(expected[i])); + i++; + } + ASSERT_EQUAL(i, size_t(3)); + return true; + }())); + + SUBTEST("tight RS around mixed containers (no whitespace anywhere)", ([&]() { + // Object, array, object - all back-to-back with only RS between. + auto input = simdjson::padded_string("\x1e{\"a\":1}\x1e[1,2]\x1e{\"b\":2}"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + auto it = stream.begin(); + + // doc 0: {"a":1} + ASSERT_EQUAL(it.current_index(), size_t(1)); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + int64_t a; ASSERT_SUCCESS(doc["a"].get(a)); + ASSERT_EQUAL(a, int64_t(1)); + } + ++it; + + // doc 1: [1,2] + ASSERT_EQUAL(it.current_index(), size_t(9)); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc.get_array().get(arr)); + size_t arr_count = 0; + for (auto e : arr) { (void)e; arr_count++; } + ASSERT_EQUAL(arr_count, size_t(2)); + } + ++it; + + // doc 2: {"b":2} + ASSERT_EQUAL(it.current_index(), size_t(15)); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + int64_t b; ASSERT_SUCCESS(doc["b"].get(b)); + ASSERT_EQUAL(b, int64_t(2)); + } + return true; + }())); + + SUBTEST("tight RS multibatch: 100 objects no whitespace, bs=32", ([&]() { + // 100 tight objects `\x1e{"i":0}\x1e{"i":1}...` with a small + // batch size to stress the multibatch path. Objects are + // delimited by their closing '}' so this case works without + // any whitespace. + constexpr size_t N = 100; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + bytes += "{\"i\":"; + bytes += std::to_string(i); + bytes += "}"; + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 32, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; + ASSERT_SUCCESS(doc["i"].get(v)); + ASSERT_EQUAL(v, int64_t(i)); + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("spaced RS multibatch: 100 int scalars RS+space+value+LF, bs=32", ([&]() { + // 100 int scalars shaped as `\x1e \n` each. Small batch + // size (32) forces ~16 stage1 invocations over the ~500 byte + // input. + constexpr size_t N = 100; + std::string bytes; + for (size_t i = 0; i < N; i++) { + bytes += '\x1e'; + bytes += ' '; + bytes += std::to_string(i); + bytes += '\n'; + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 32, simdjson::stream_format::json_sequence).get(stream)); + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; + ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(i)); + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("leading/trailing/repeated RS separators around scalars", ([&]() { + // RFC 7464 says each JSON text is preceded by exactly one RS. + // The sensible interpretation of degenerate inputs: + // - leading RS-RS (empty record before the first scalar): skip + // - repeated RSes between two scalars (empty intermediate + // record): skip the empty record, both scalars reported + // - trailing bare RS with no payload: skip, no phantom doc + // Layout: RS(0) RS(1) 1(2) LF(3) RS(4) RS(5) RS(6) 2(7) LF(8) RS(9) + auto input = simdjson::padded_string("\x1e\x1e" "1\n" "\x1e\x1e\x1e" "2\n" "\x1e"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream)); + int64_t expected[] = {1, 2}; + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < 2); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, expected[i]); + i++; + } + ASSERT_EQUAL(i, size_t(2)); + return true; + }())); + TEST_SUCCEED(); } @@ -1247,6 +1803,405 @@ namespace document_stream_tests { return true; }())); + SUBTEST("scalar quintet 1,2,\"x\",true,null", ([&]() { + // Five comma-delimited scalars: number, number, string, true, null. + // Layout: 1(0) ,(1) 2(2) ,(3) "(4) x(5) "(6) ,(7) t(8) r(9) u(10) e(11) ,(12) n(13) u(14) l(15) l(16) + // + // Note on source(): the comma_delimited filter strips root-level + // commas from structural_indexes but the bytes remain in the + // buffer. source() for a scalar slices [current_index, + // next_doc_index) and strips trailing whitespace, so it + // currently includes the trailing separator comma (e.g. doc 0 + // source = "1,"). The assertions below lock down current_index() + // and parsed values strictly, and accept source() either with + // or without a single trailing comma. + auto input = R"(1,2,"x",true,null)"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream)); + auto it = stream.begin(); + + auto src_matches = [](std::string_view src, std::string_view want) { + return src == want || src == std::string(want) + ","; + }; + + // doc 0: 1 + ASSERT_EQUAL(it.current_index(), size_t(0)); + ASSERT_TRUE(src_matches(it.source(), "1")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(1)); + } + ++it; + + // doc 1: 2 + ASSERT_EQUAL(it.current_index(), size_t(2)); + ASSERT_TRUE(src_matches(it.source(), "2")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(2)); + } + ++it; + + // doc 2: "x" + ASSERT_EQUAL(it.current_index(), size_t(4)); + ASSERT_TRUE(src_matches(it.source(), "\"x\"")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + std::string_view sv; ASSERT_SUCCESS(doc.get_string().get(sv)); + ASSERT_EQUAL(sv, std::string_view("x")); + } + ++it; + + // doc 3: true + ASSERT_EQUAL(it.current_index(), size_t(8)); + ASSERT_TRUE(src_matches(it.source(), "true")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + bool v; ASSERT_SUCCESS(doc.get_bool().get(v)); + ASSERT_TRUE(v); + } + ++it; + + // doc 4: null (last doc, no trailing comma in source either way) + ASSERT_EQUAL(it.current_index(), size_t(13)); + ASSERT_EQUAL(it.source(), std::string_view("null")); + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + bool is_null; + ASSERT_SUCCESS(doc.is_null().get(is_null)); + ASSERT_TRUE(is_null); + } + return true; + }())); + + SUBTEST("scalar quintet multibatch (small batch_size)", ([&]() { + // Same input as the scalar quintet test, but batch_size=8 forces + // the stream through multiple stage1 invocations. Behavior must + // match the single-batch case for current_index. + auto input = R"(1,2,"x",true,null)"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, 8, simdjson::stream_format::comma_delimited).get(stream)); + size_t count = 0; + const size_t expected_idx[5] = {0, 2, 4, 8, 13}; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc = *it; + ASSERT_SUCCESS(doc.error()); + ASSERT_TRUE(count < 5); + ASSERT_EQUAL(it.current_index(), expected_idx[count]); + count++; + } + ASSERT_EQUAL(count, size_t(5)); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch", ([&]() { + // Stress the windowing / multi-stage1 path with 256 comma- + // delimited scalar documents rotating through all four scalar + // types. A small batch_size (64) forces ~20 stage1 invocations + // over ~1.3 KB of input. Every document must parse and appear + // in order. + constexpr size_t N = 256; + std::string bytes; + for (size_t i = 0; i < N; i++) { + if (i > 0) { bytes += ','; } + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + } + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::comma_delimited).get(stream)); + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get_bool().get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + bool is_null; + ASSERT_SUCCESS(doc.is_null().get(is_null)); + ASSERT_TRUE(is_null); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() { + // Same as the large stress test, but with a leading comma, + // extra commas sprinkled every 5 documents, and two trailing + // commas. The comma_delimited filter is lenient about + // degenerate separators, so this test is STRICT: exactly N + // documents must still come out, in order, with correct + // values. + constexpr size_t N = 256; + std::string bytes; + bytes += ','; // leading comma + for (size_t i = 0; i < N; i++) { + if (i > 0) { bytes += ','; } + if (i % 5 == 0 && i > 0) { bytes += ','; } // extra comma + switch (i % 4) { + case 0: bytes += std::to_string(i); break; + case 1: bytes += (i & 1) ? "true" : "false"; break; + case 2: bytes += "\"s" + std::to_string(i) + "\""; break; + case 3: bytes += "null"; break; + } + } + bytes += ",,"; // trailing commas + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::comma_delimited).get(stream)); + size_t i = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(i < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + switch (i % 4) { + case 0: { + int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(i)); + break; + } + case 1: { + bool v; ASSERT_SUCCESS(doc.get_bool().get(v)); + ASSERT_EQUAL(v, bool(i & 1)); + break; + } + case 2: { + std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v)); + std::string want = "s" + std::to_string(i); + ASSERT_EQUAL(v, std::string_view(want)); + break; + } + case 3: { + bool is_null; + ASSERT_SUCCESS(doc.is_null().get(is_null)); + ASSERT_TRUE(is_null); + break; + } + } + i++; + } + ASSERT_EQUAL(i, N); + return true; + }())); + + SUBTEST("leading/trailing/repeated commas around scalars", ([&]() { + // Comma-delimited filter is expected to be lenient: leading + // commas, trailing commas, and repeated commas between scalars + // all collapse to single separators. The three real scalars + // (1, 2, "x") must be reported with no phantom documents. + // Layout: ,(0) 1(1) ,(2) ,(3) 2(4) ,(5) ,(6) "(7) x(8) "(9) ,(10) ,(11) + auto input = R"(,1,,2,,"x",,)"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream)); + size_t count = 0; + int64_t i1 = -1, i2 = -1; + std::string_view sx; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + if (count == 0) { ASSERT_SUCCESS(doc.get_int64().get(i1)); } + else if (count == 1) { ASSERT_SUCCESS(doc.get_int64().get(i2)); } + else if (count == 2) { ASSERT_SUCCESS(doc.get_string().get(sx)); } + count++; + } + ASSERT_EQUAL(count, size_t(3)); + ASSERT_EQUAL(i1, int64_t(1)); + ASSERT_EQUAL(i2, int64_t(2)); + ASSERT_EQUAL(sx, std::string_view("x")); + return true; + }())); + + // -------- stream_format::comma_delimited_array -------- + // The comma_delimited_array mode strips the outer [ ] (plus any + // surrounding JSON whitespace) and then delegates to comma_delimited. + + SUBTEST("comma_delimited_array basic [1,2,3]", ([&]() { + auto input = R"([1,2,3])"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + int64_t got[3] = {0, 0, 0}; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(count < 3); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + ASSERT_SUCCESS(doc.get_int64().get(got[count])); + count++; + } + ASSERT_EQUAL(count, size_t(3)); + ASSERT_EQUAL(got[0], int64_t(1)); + ASSERT_EQUAL(got[1], int64_t(2)); + ASSERT_EQUAL(got[2], int64_t(3)); + return true; + }())); + + SUBTEST("comma_delimited_array mixed scalar types", ([&]() { + auto input = R"([1, "a", true, null, {"x":1}])"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; } + ASSERT_EQUAL(count, size_t(5)); + return true; + }())); + + SUBTEST("comma_delimited_array empty []", ([&]() { + auto input = R"([])"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { (void)doc; count++; } + ASSERT_EQUAL(count, size_t(0)); + return true; + }())); + + SUBTEST("comma_delimited_array empty with interior whitespace [ ]", ([&]() { + auto input = simdjson::padded_string("[ \t\n]"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto doc : stream) { (void)doc; count++; } + ASSERT_EQUAL(count, size_t(0)); + return true; + }())); + + SUBTEST("comma_delimited_array surrounding whitespace stripped", ([&]() { + // All four JSON whitespace characters before the '[' and after + // the ']'. + auto input = simdjson::padded_string(" \t\n\r[ 1, 2, 3 ] \t\n\r"s); + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + int64_t sum = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; + ASSERT_SUCCESS(doc.get_int64().get(v)); + sum += v; + count++; + } + ASSERT_EQUAL(count, size_t(3)); + ASSERT_EQUAL(sum, int64_t(6)); + return true; + }())); + + SUBTEST("comma_delimited_array nested commas preserved", ([&]() { + auto input = R"([{"a":1,"b":2},{"c":[3,4,5]}])"_padded; + ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream)); + auto it = stream.begin(); + // doc 0: {"a":1,"b":2} + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + int64_t a, b; + ASSERT_SUCCESS(doc["a"].get(a)); + ASSERT_SUCCESS(doc["b"].get(b)); + ASSERT_EQUAL(a, int64_t(1)); + ASSERT_EQUAL(b, int64_t(2)); + } + ++it; + // doc 1: {"c":[3,4,5]} + { + ondemand::document_reference doc; + ASSERT_SUCCESS((*it).get(doc)); + ondemand::array arr; + ASSERT_SUCCESS(doc["c"].get_array().get(arr)); + size_t arr_count = 0; + for (auto e : arr) { (void)e; arr_count++; } + ASSERT_EQUAL(arr_count, size_t(3)); + } + return true; + }())); + + SUBTEST("comma_delimited_array missing leading bracket is TAPE_ERROR", ([&]() { + auto input = R"(1,2,3)"_padded; + auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array missing trailing bracket is TAPE_ERROR", ([&]() { + auto input = R"([1,2,3)"_padded; + auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array empty input is TAPE_ERROR", ([&]() { + auto input = simdjson::padded_string(""s); + auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array whitespace-only input is TAPE_ERROR", ([&]() { + auto input = simdjson::padded_string(" \t\n"s); + auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream); + ASSERT_ERROR(err, simdjson::TAPE_ERROR); + return true; + }())); + + SUBTEST("comma_delimited_array multibatch large array", ([&]() { + // Generate [0,1,2,...,N-1] and iterate with a small batch_size + // so the stream runs stage1 across many windows. All N values + // must come out in order with no phantom documents. + constexpr size_t N = 256; + std::string bytes = "["; + for (size_t i = 0; i < N; i++) { + if (i > 0) { bytes += ','; } + bytes += std::to_string(i); + } + bytes += "]"; + auto input = simdjson::padded_string(bytes); + ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::comma_delimited_array).get(stream)); + size_t count = 0; + for (auto it = stream.begin(); it != stream.end(); ++it) { + auto doc_res = *it; + ASSERT_SUCCESS(doc_res.error()); + ASSERT_TRUE(count < N); + ondemand::document_reference doc; + ASSERT_SUCCESS(doc_res.get(doc)); + int64_t v; + ASSERT_SUCCESS(doc.get_int64().get(v)); + ASSERT_EQUAL(v, int64_t(count)); + count++; + } + ASSERT_EQUAL(count, N); + return true; + }())); + TEST_SUCCEED(); }