mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
finishing up the new streaming (#2674)
* finishing up the new streaming * fixed silly warnings. * tweak * adding more tests * minor fixes * more fixes
This commit is contained in:
Daniel Lemire
@@ -366,9 +366,43 @@ The `stream_format` enum has the following values:
|
||||
- `stream_format::whitespace_delimited` (default): Standard NDJSON/JSON Lines format
|
||||
- `stream_format::json_sequence`: RFC 7464 format with RS delimiters
|
||||
- `stream_format::comma_delimited`: Comma-separated JSON documents
|
||||
- `stream_format::comma_delimited_array`: A single JSON array whose elements are iterated as comma-delimited documents (see below)
|
||||
|
||||
The trailing LF after each JSON text is optional but recommended by the RFC for robustness.
|
||||
|
||||
JSON Array As A Document Stream
|
||||
-------------------------------
|
||||
|
||||
Sometimes an input is a single, well-formed JSON array — `[{"a":1},{"b":2},{"c":3}]` — but you want to iterate its elements one at a time without materializing the whole array. Use `stream_format::comma_delimited_array`:
|
||||
|
||||
```cpp
|
||||
auto json = R"([{"a":1},{"b":2},{"c":3}])"_padded;
|
||||
ondemand::parser parser;
|
||||
ondemand::document_stream stream;
|
||||
auto error = parser.iterate_many(json, ondemand::DEFAULT_BATCH_SIZE,
|
||||
simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
if (error) { std::cerr << error << std::endl; return; }
|
||||
for (auto doc : stream) {
|
||||
std::cout << doc << std::endl;
|
||||
}
|
||||
// Prints: {"a":1}
|
||||
// {"b":2}
|
||||
// {"c":3}
|
||||
```
|
||||
|
||||
The parser strips the outer `[` and `]` plus any surrounding JSON whitespace (space, tab, LF, CR) and then behaves exactly like `stream_format::comma_delimited` over the remaining bytes. All comma-delimited features are inherited: multi-batch processing, threading, mixed scalar types, and nested commas preserved inside inner objects and arrays.
|
||||
|
||||
```cpp
|
||||
// All of these work:
|
||||
auto a = R"([1, "x", true, null, {"k":"v"}, [1,2]])"_padded; // mixed scalars
|
||||
auto b = R"( [ 1, 2, 3 ] )"_padded; // whitespace
|
||||
auto c = R"([])"_padded; // empty array → 0 docs
|
||||
```
|
||||
|
||||
If the input is not a well-formed outer array (missing `[`, missing `]`, or empty / all-whitespace), `iterate_many` returns `TAPE_ERROR`. Content **inside** the array is not validated up front — individual document parse errors surface when you iterate, just like `comma_delimited`.
|
||||
|
||||
Positions reported via `current_index()` are relative to the **stripped** buffer (the bytes between `[` and `]`), not the original input, for consistency with the existing BOM-stripping behavior.
|
||||
|
||||
|
||||
C++20 features
|
||||
--------------------
|
||||
|
||||
@@ -290,6 +290,7 @@ The `stream_format` enum has the following values:
|
||||
- `stream_format::whitespace_delimited` (default): Standard NDJSON/JSON Lines format
|
||||
- `stream_format::json_sequence`: RFC 7464 format with RS delimiters
|
||||
- `stream_format::comma_delimited`: Comma-separated JSON documents
|
||||
- `stream_format::comma_delimited_array`: A single JSON array whose elements are iterated as comma-delimited documents (see below)
|
||||
|
||||
The trailing LF after each JSON text is optional but recommended by the RFC for robustness.
|
||||
|
||||
@@ -331,3 +332,36 @@ On-Demand comma-separated mode. Leading commas, trailing commas, and repeated
|
||||
commas are treated as empty separators rather than documents.
|
||||
|
||||
Unlike the legacy `allow_comma_separated` parameter, `stream_format::comma_delimited` supports multi-batch processing and threading for optimal performance on large files.
|
||||
|
||||
JSON Array As A Document Stream
|
||||
-------------------------------
|
||||
|
||||
Sometimes an input is a single, well-formed JSON array — `[{"a":1},{"b":2},{"c":3}]` — but you want to iterate its elements one at a time without materializing the whole array. Use `stream_format::comma_delimited_array`:
|
||||
|
||||
```cpp
|
||||
auto json = R"([{"a":1},{"b":2},{"c":3}])"_padded;
|
||||
simdjson::dom::parser parser;
|
||||
simdjson::dom::document_stream stream;
|
||||
auto error = parser.parse_many(json, simdjson::dom::DEFAULT_BATCH_SIZE,
|
||||
simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
if (error) { std::cerr << error << std::endl; return; }
|
||||
for (auto doc : stream) {
|
||||
std::cout << doc << std::endl;
|
||||
}
|
||||
// Prints: {"a":1}
|
||||
// {"b":2}
|
||||
// {"c":3}
|
||||
```
|
||||
|
||||
The parser strips the outer `[` and `]` plus any surrounding JSON whitespace (space, tab, LF, CR) and then behaves exactly like `stream_format::comma_delimited` over the remaining bytes. All comma-delimited features are inherited: multi-batch processing, threading, mixed scalar types, and nested commas preserved inside inner objects and arrays.
|
||||
|
||||
```cpp
|
||||
// All of these work:
|
||||
auto a = R"([1, "x", true, null, {"k":"v"}, [1,2]])"_padded; // mixed scalars
|
||||
auto b = R"( [ 1, 2, 3 ] )"_padded; // whitespace
|
||||
auto c = R"([])"_padded; // empty array → 0 docs
|
||||
```
|
||||
|
||||
If the input is not a well-formed outer array (missing `[`, missing `]`, or empty / all-whitespace), `parse_many` returns `TAPE_ERROR`. Content **inside** the array is not validated up front — individual document parse errors surface when you iterate, just like `comma_delimited`.
|
||||
|
||||
Positions reported via `current_index()` are relative to the **stripped** buffer (the bytes between `[` and `]`), not the original input, for consistency with the existing BOM-stripping behavior.
|
||||
|
||||
@@ -56,7 +56,13 @@ enum class stage1_mode;
|
||||
enum class stream_format {
|
||||
whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL)
|
||||
json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited)
|
||||
comma_delimited ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`)
|
||||
comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`)
|
||||
comma_delimited_array ///< A single JSON array whose elements are iterated as
|
||||
///< comma-separated documents (e.g., `[{...},{...},{...}]`).
|
||||
///< The parser strips the outer `[` / `]` plus any
|
||||
///< surrounding JSON whitespace (space, tab, LF, CR)
|
||||
///< and then behaves like `comma_delimited` over the
|
||||
///< remaining bytes.
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
|
||||
@@ -227,7 +227,14 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
|
||||
} else {
|
||||
size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index];
|
||||
size_t svlen = next_doc_index - current_index();
|
||||
while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
|
||||
// Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464 json_sequence
|
||||
// mode the scanner classifies RS as a scalar character, so an RS-prefixed
|
||||
// scalar document (number/true/false/null/string) has no closing structural
|
||||
// index and the slice runs all the way up to the next document's RS. RS
|
||||
// cannot legally appear in a JSON value at the source level (control
|
||||
// characters in strings must be escaped as \u001E), so stripping it is
|
||||
// safe in every stream_format.
|
||||
while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0' || static_cast<uint8_t>(start[svlen-1]) == 0x1E)) {
|
||||
svlen--;
|
||||
}
|
||||
return std::string_view(start, svlen);
|
||||
|
||||
@@ -188,6 +188,24 @@ inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, s
|
||||
buf += 3;
|
||||
len -= 3;
|
||||
}
|
||||
if (format == stream_format::comma_delimited_array) {
|
||||
// Strip leading JSON whitespace.
|
||||
while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) {
|
||||
buf++; len--;
|
||||
}
|
||||
// Expect the opening '['.
|
||||
if (len == 0 || buf[0] != '[') { return TAPE_ERROR; }
|
||||
buf++; len--;
|
||||
// Strip trailing JSON whitespace.
|
||||
while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) {
|
||||
len--;
|
||||
}
|
||||
// Expect the closing ']'.
|
||||
if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; }
|
||||
len--;
|
||||
// Fall through to comma_delimited over the array contents.
|
||||
format = stream_format::comma_delimited;
|
||||
}
|
||||
return document_stream(*this, buf, len, batch_size, format);
|
||||
}
|
||||
inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept {
|
||||
|
||||
@@ -384,14 +384,21 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc
|
||||
depth--;
|
||||
break;
|
||||
default: // Scalar value document
|
||||
// TODO: We could remove trailing whitespaces
|
||||
// This returns a string spanning from start of value to the beginning of the next document (excluded)
|
||||
{
|
||||
auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index];
|
||||
// normally the length would be next_index - current_index() - 1, except for the last document
|
||||
size_t svlen = next_index - current_index();
|
||||
const char *start = reinterpret_cast<const char*>(stream->buf) + current_index();
|
||||
while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) {
|
||||
// Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464
|
||||
// json_sequence mode the scanner classifies RS as a scalar
|
||||
// character, so an RS-prefixed scalar document (number / true /
|
||||
// false / null / string) has no closing structural index and the
|
||||
// slice runs all the way up to the next document's RS. RS cannot
|
||||
// legally appear in a JSON value at the source level (control
|
||||
// characters in strings must be escaped as \u001E), so stripping
|
||||
// it is safe in every stream_format.
|
||||
while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0' || static_cast<uint8_t>(start[svlen-1]) == 0x1E)) {
|
||||
svlen--;
|
||||
}
|
||||
return std::string_view(start, svlen);
|
||||
|
||||
@@ -201,6 +201,24 @@ inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf,
|
||||
buf += 3;
|
||||
len -= 3;
|
||||
}
|
||||
if (format == stream_format::comma_delimited_array) {
|
||||
// Strip leading JSON whitespace.
|
||||
while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) {
|
||||
buf++; len--;
|
||||
}
|
||||
// Expect the opening '['.
|
||||
if (len == 0 || buf[0] != '[') { return TAPE_ERROR; }
|
||||
buf++; len--;
|
||||
// Strip trailing JSON whitespace.
|
||||
while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) {
|
||||
len--;
|
||||
}
|
||||
// Expect the closing ']'.
|
||||
if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; }
|
||||
len--;
|
||||
// Fall through to comma_delimited over the array contents.
|
||||
format = stream_format::comma_delimited;
|
||||
}
|
||||
return document_stream(*this, buf, len, batch_size, false, format);
|
||||
}
|
||||
inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept {
|
||||
|
||||
+8
-2
@@ -226,9 +226,15 @@ simdjson_warn_unused simdjson_inline error_code scan() {
|
||||
add_structural();
|
||||
// Primitive or invalid character (invalid characters will be checked in stage 2)
|
||||
} else {
|
||||
// Anything else, add the structural and go until we find the next one
|
||||
// Anything else, add the structural and go until we find the next one.
|
||||
// We also stop on RS (0x1E) so that RFC 7464 json_sequence inputs
|
||||
// like `\x1e"a"\x1e"b"` produce a separate structural for each RS
|
||||
// rather than being absorbed into a single primitive run. RS is a
|
||||
// control character that is invalid in normal JSON, so breaking
|
||||
// the run here has no effect on well-formed non-json_sequence
|
||||
// inputs.
|
||||
add_structural();
|
||||
while (idx+1<len && !char_is_space_or_operator(buf[idx+1])) {
|
||||
while (idx+1<len && !char_is_space_or_operator(buf[idx+1]) && buf[idx+1] != 0x1e) {
|
||||
idx++;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -147,28 +147,66 @@ simdjson_inline uint32_t find_next_document_index_json_sequence(
|
||||
for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) {
|
||||
const uint32_t pos = parser.structural_indexes[read_idx];
|
||||
if (parser.buf[pos] == 0x1E) {
|
||||
// This is an RS character - find the actual JSON value start
|
||||
// This is an RS character - find the actual JSON value start.
|
||||
last_rs_pos = pos;
|
||||
rs_count++;
|
||||
// Skip RS and any whitespace to find the actual value start
|
||||
// Skip past this RS and any whitespace *and any additional RSes*
|
||||
// to locate the real value. Consecutive RSes are degenerate
|
||||
// "empty records" per RFC 7464; we collapse them here. They do
|
||||
// not always appear as separate entries in structural_indexes
|
||||
// because the scanner groups runs of adjacent non-whitespace
|
||||
// scalar bytes (including RS) into a single scalar start.
|
||||
uint32_t value_start = pos + 1;
|
||||
while (value_start < len) {
|
||||
const uint8_t c = parser.buf[value_start];
|
||||
if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { value_start++; }
|
||||
else { break; }
|
||||
if (c == ' ' || c == '\t' || c == '\n' || c == '\r') {
|
||||
value_start++;
|
||||
} else if (c == 0x1E) {
|
||||
// Collapsed empty record. Still count it so rs_count reflects
|
||||
// the true number of record markers and last_rs_pos tracks
|
||||
// the final one.
|
||||
last_rs_pos = value_start;
|
||||
rs_count++;
|
||||
value_start++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Check if the value start is an operator (already in structural_indexes),
|
||||
// another RS (empty record), or a scalar (needs to be added).
|
||||
// If the scanner emitted additional structurals inside the
|
||||
// whitespace+RS run we just walked over (i.e., isolated RSes
|
||||
// separated by whitespace), skip past them so we do not
|
||||
// double-count or double-emit.
|
||||
while (read_idx + 1 < parser.n_structural_indexes &&
|
||||
parser.structural_indexes[read_idx + 1] < value_start) {
|
||||
read_idx++;
|
||||
}
|
||||
// Check if the value start is an operator (always present in
|
||||
// scanner structural_indexes) or a scalar-like start (which may
|
||||
// be missing from structural_indexes and must be added here).
|
||||
// Note: '"' is NOT always in structural_indexes. The scanner
|
||||
// classifies '"' as a scalar character and emits it as a
|
||||
// structural only when it is a *scalar start* (preceded by
|
||||
// whitespace or an operator). When '"' immediately follows an
|
||||
// RS (which the scanner also classifies as scalar), it is
|
||||
// treated as a scalar continuation and not emitted - so we
|
||||
// must add it here just like any other scalar value.
|
||||
if (value_start < len) {
|
||||
const uint8_t c = parser.buf[value_start];
|
||||
const bool is_operator = (c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',');
|
||||
const bool is_rs = (c == 0x1E);
|
||||
if (!is_operator && !is_rs) {
|
||||
// Scalar value - add its position since scanner missed it
|
||||
const bool is_operator =
|
||||
(c == '{' || c == '}' || c == '[' || c == ']' ||
|
||||
c == ':' || c == ',');
|
||||
// If the next scanner structural is exactly at value_start,
|
||||
// the scanner already emitted it (it followed whitespace) and
|
||||
// we must not add a duplicate - a subsequent iteration will
|
||||
// copy it into write_idx.
|
||||
const bool already_emitted =
|
||||
(read_idx + 1 < parser.n_structural_indexes &&
|
||||
parser.structural_indexes[read_idx + 1] == value_start);
|
||||
if (!is_operator && !already_emitted) {
|
||||
// Scalar value (number/true/false/null/string) - add its
|
||||
// position since scanner missed it.
|
||||
parser.structural_indexes[write_idx++] = value_start;
|
||||
}
|
||||
// For operators, they will appear later in structural_indexes
|
||||
// For RS, it will be processed in a subsequent iteration
|
||||
}
|
||||
} else {
|
||||
// Not RS, copy to output
|
||||
|
||||
@@ -4,6 +4,10 @@ include(${PROJECT_SOURCE_DIR}/cmake/add_cpp_test.cmake)
|
||||
add_subdirectory(dom)
|
||||
add_subdirectory(ondemand)
|
||||
|
||||
# compilation_failure_tests is added before the global link_libraries(simdjson) so that
|
||||
# multiple_include/myexe does not receive a duplicate simdjson entry alongside the one
|
||||
# already propagated by mylib PUBLIC simdjson::simdjson.
|
||||
add_subdirectory(compilation_failure_tests)
|
||||
|
||||
# All remaining tests link with simdjson proper
|
||||
link_libraries(simdjson)
|
||||
@@ -27,6 +31,5 @@ endif()
|
||||
# SIMDJSON_FORCE_IMPLEMENTATION, so we know we're testing what we think we're testing
|
||||
add_cpp_test(checkimplementation LABELS other per_implementation)
|
||||
|
||||
add_subdirectory(compilation_failure_tests)
|
||||
add_subdirectory(builder)
|
||||
add_subdirectory(compile_time)
|
||||
@@ -14,6 +14,17 @@ function(add_dual_compile_test TEST_NAME)
|
||||
endfunction(add_dual_compile_test)
|
||||
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
# Add multiple_include BEFORE link_libraries(simdjson) so that myexe only receives
|
||||
# simdjson once (through mylib PUBLIC simdjson::simdjson) and not a second time from
|
||||
# a directory-wide link entry.
|
||||
add_subdirectory(multiple_include)
|
||||
endif()
|
||||
|
||||
# Dual-compile tests build executables that reference simdjson symbols, so they need
|
||||
# the library linked in.
|
||||
link_libraries(simdjson)
|
||||
|
||||
add_dual_compile_test(example_compiletest)
|
||||
# These don't compile with exceptions off
|
||||
if (SIMDJSON_EXCEPTIONS)
|
||||
@@ -24,9 +35,4 @@ if (SIMDJSON_EXCEPTIONS)
|
||||
add_dual_compile_test(dangling_parser_parse_stdstring)
|
||||
add_dual_compile_test(dangling_parser_parse_padstring)
|
||||
add_dual_compile_test(unsafe_parse_many)
|
||||
endif()
|
||||
|
||||
if(NOT BUILD_SHARED_LIBS)
|
||||
# We only check that it builds
|
||||
add_subdirectory(multiple_include)
|
||||
endif()
|
||||
@@ -0,0 +1,230 @@
|
||||
#ifndef SIMDJSON_TESTS_DOCUMENT_STREAM_FUZZ_TEST_COMMON_H
|
||||
#define SIMDJSON_TESTS_DOCUMENT_STREAM_FUZZ_TEST_COMMON_H
|
||||
|
||||
#include <cstdint>
|
||||
#include <random>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "simdjson.h"
|
||||
|
||||
namespace document_stream_fuzz {
|
||||
|
||||
constexpr uint64_t fixed_seed = 0x5eed1234ULL;
|
||||
constexpr size_t batch_size = 512;
|
||||
constexpr size_t minimum_total_bytes = 4096;
|
||||
constexpr size_t minimum_document_count = 128;
|
||||
|
||||
struct stream_case {
|
||||
const char *name;
|
||||
simdjson::stream_format format;
|
||||
std::string input;
|
||||
std::vector<std::string> expected_documents;
|
||||
};
|
||||
|
||||
inline char random_char(std::mt19937_64 &rng) {
|
||||
static constexpr char alphabet[] =
|
||||
"abcdefghijklmnopqrstuvwxyz"
|
||||
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
"0123456789 _-/.";
|
||||
std::uniform_int_distribution<size_t> dist(0, sizeof(alphabet) - 2);
|
||||
return alphabet[dist(rng)];
|
||||
}
|
||||
|
||||
inline std::string make_ascii_text(std::mt19937_64 &rng, size_t min_length, size_t max_length) {
|
||||
std::uniform_int_distribution<size_t> length_dist(min_length, max_length);
|
||||
const size_t length = length_dist(rng);
|
||||
std::string text;
|
||||
text.reserve(length);
|
||||
for (size_t i = 0; i < length; i++) {
|
||||
text.push_back(random_char(rng));
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
inline std::string quote_json_string(const std::string &text) {
|
||||
std::string quoted;
|
||||
quoted.reserve(text.size() + 2);
|
||||
quoted.push_back('"');
|
||||
for (char ch : text) {
|
||||
if (ch == '\\' || ch == '"') {
|
||||
quoted.push_back('\\');
|
||||
}
|
||||
quoted.push_back(ch);
|
||||
}
|
||||
quoted.push_back('"');
|
||||
return quoted;
|
||||
}
|
||||
|
||||
inline std::string make_integer_literal(std::mt19937_64 &rng) {
|
||||
std::uniform_int_distribution<int64_t> dist(-1000000, 1000000);
|
||||
return std::to_string(dist(rng));
|
||||
}
|
||||
|
||||
inline std::string make_float_literal(std::mt19937_64 &rng) {
|
||||
std::uniform_int_distribution<int64_t> whole_dist(-250000, 250000);
|
||||
std::uniform_int_distribution<int64_t> fraction_dist(1, 9999);
|
||||
return std::to_string(whole_dist(rng)) + "." + std::to_string(fraction_dist(rng));
|
||||
}
|
||||
|
||||
inline std::string make_scalar(std::mt19937_64 &rng) {
|
||||
std::uniform_int_distribution<int> type_dist(0, 4);
|
||||
switch (type_dist(rng)) {
|
||||
case 0: return quote_json_string(make_ascii_text(rng, 0, 20));
|
||||
case 1: return make_integer_literal(rng);
|
||||
case 2: return make_float_literal(rng);
|
||||
case 3: return (rng() & 1) ? "true" : "false";
|
||||
default: return "null";
|
||||
}
|
||||
}
|
||||
|
||||
inline std::string make_value(std::mt19937_64 &rng, int depth);
|
||||
|
||||
inline std::string make_array(std::mt19937_64 &rng, int depth) {
|
||||
std::uniform_int_distribution<int> count_dist(0, depth == 0 ? 5 : 3);
|
||||
const int count = count_dist(rng);
|
||||
std::string out = "[";
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (i > 0) {
|
||||
out.push_back(',');
|
||||
}
|
||||
out += make_value(rng, depth + 1);
|
||||
}
|
||||
out.push_back(']');
|
||||
return out;
|
||||
}
|
||||
|
||||
inline std::string make_object(std::mt19937_64 &rng, int depth) {
|
||||
std::uniform_int_distribution<int> count_dist(0, depth == 0 ? 5 : 3);
|
||||
const int count = count_dist(rng);
|
||||
std::string out = "{";
|
||||
for (int i = 0; i < count; i++) {
|
||||
if (i > 0) {
|
||||
out.push_back(',');
|
||||
}
|
||||
std::string key = "k";
|
||||
key += std::to_string(depth);
|
||||
key += '_';
|
||||
key += std::to_string(i);
|
||||
key += '_';
|
||||
key += make_ascii_text(rng, 1, 6);
|
||||
out += quote_json_string(key);
|
||||
out.push_back(':');
|
||||
out += make_value(rng, depth + 1);
|
||||
}
|
||||
out.push_back('}');
|
||||
return out;
|
||||
}
|
||||
|
||||
inline std::string make_value(std::mt19937_64 &rng, int depth) {
|
||||
if (depth >= 3) {
|
||||
return make_scalar(rng);
|
||||
}
|
||||
std::uniform_int_distribution<int> type_dist(0, 6);
|
||||
switch (type_dist(rng)) {
|
||||
case 0: return quote_json_string(make_ascii_text(rng, 0, 20));
|
||||
case 1: return make_integer_literal(rng);
|
||||
case 2: return make_float_literal(rng);
|
||||
case 3: return (rng() & 1) ? "true" : "false";
|
||||
case 4: return "null";
|
||||
case 5: return make_array(rng, depth);
|
||||
default: return make_object(rng, depth);
|
||||
}
|
||||
}
|
||||
|
||||
inline std::vector<std::string> make_documents() {
|
||||
std::vector<std::string> docs = {
|
||||
"0",
|
||||
"-17",
|
||||
"3.125",
|
||||
"true",
|
||||
"false",
|
||||
"null",
|
||||
"\"alpha beta\"",
|
||||
"[]",
|
||||
"[1,true,\"x\"]",
|
||||
"{}",
|
||||
"{\"a\":1,\"b\":[2,3],\"c\":{\"d\":false}}"
|
||||
};
|
||||
|
||||
size_t total_bytes = 0;
|
||||
for (const auto &doc : docs) {
|
||||
total_bytes += doc.size();
|
||||
}
|
||||
|
||||
std::mt19937_64 rng(fixed_seed);
|
||||
while (docs.size() < minimum_document_count || total_bytes < minimum_total_bytes) {
|
||||
docs.push_back(make_value(rng, 0));
|
||||
total_bytes += docs.back().size();
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
inline std::vector<std::string> make_wrapped_documents(const std::vector<std::string> &docs) {
|
||||
std::vector<std::string> wrapped;
|
||||
wrapped.reserve(docs.size());
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
wrapped.push_back("{\"id\":" + std::to_string(i) + ",\"value\":" + docs[i] + "}");
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
inline std::string build_whitespace_input(const std::vector<std::string> &docs) {
|
||||
static const char *separators[] = {" ", "\n", "\r\n", "\t", " \n\t", "\r\t "};
|
||||
std::string out = " \n\t";
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
out += docs[i];
|
||||
if (i + 1 < docs.size()) {
|
||||
out += separators[i % (sizeof(separators) / sizeof(separators[0]))];
|
||||
}
|
||||
}
|
||||
out += "\n\t ";
|
||||
return out;
|
||||
}
|
||||
|
||||
inline std::string build_json_sequence_input(const std::vector<std::string> &docs) {
|
||||
std::string out;
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
out.push_back('\x1e');
|
||||
out += docs[i];
|
||||
out.push_back('\n');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
inline std::string build_comma_delimited_input(const std::vector<std::string> &docs) {
|
||||
std::string out;
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
if (i > 0) {
|
||||
out += (i % 3 == 0) ? ",\n" : ", ";
|
||||
}
|
||||
out += docs[i];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
inline std::string build_comma_delimited_array_input(const std::vector<std::string> &docs) {
|
||||
std::string out = " \t\n[";
|
||||
for (size_t i = 0; i < docs.size(); i++) {
|
||||
if (i > 0) {
|
||||
out += (i % 4 == 0) ? ",\n" : ", ";
|
||||
}
|
||||
out += docs[i];
|
||||
}
|
||||
out += "]\r\n";
|
||||
return out;
|
||||
}
|
||||
|
||||
inline std::vector<stream_case> make_stream_cases(const std::vector<std::string> &docs) {
|
||||
std::vector<std::string> whitespace_docs = make_wrapped_documents(docs);
|
||||
return {
|
||||
{"whitespace_delimited", simdjson::stream_format::whitespace_delimited, build_whitespace_input(whitespace_docs), whitespace_docs},
|
||||
{"json_sequence", simdjson::stream_format::json_sequence, build_json_sequence_input(whitespace_docs), whitespace_docs},
|
||||
{"comma_delimited", simdjson::stream_format::comma_delimited, build_comma_delimited_input(whitespace_docs), whitespace_docs},
|
||||
{"comma_delimited_array", simdjson::stream_format::comma_delimited_array, build_comma_delimited_array_input(whitespace_docs), whitespace_docs}
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace document_stream_fuzz
|
||||
|
||||
#endif
|
||||
@@ -7,6 +7,7 @@ if(NOT SIMDJSON_LEGACY_VISUAL_STUDIO AND NOT SIMDJSON_WINDOWS_DLL)
|
||||
endif()
|
||||
add_cpp_test(basictests LABELS dom acceptance per_implementation)
|
||||
add_cpp_test(document_stream_tests LABELS dom acceptance per_implementation)
|
||||
add_cpp_test(document_stream_fuzz_tests LABELS dom acceptance per_implementation)
|
||||
add_cpp_test(document_tests LABELS dom acceptance per_implementation)
|
||||
add_cpp_test(errortests LABELS dom acceptance per_implementation)
|
||||
add_cpp_test(extracting_values_example LABELS dom acceptance per_implementation)
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "simdjson.h"
|
||||
#include "test_macros.h"
|
||||
#include "test_main.h"
|
||||
#include "document_stream_fuzz_test_common.h"
|
||||
|
||||
namespace document_stream_fuzz_tests {
|
||||
|
||||
std::string strip_stream_artifacts(simdjson::stream_format format, std::string_view text) {
|
||||
size_t start = 0;
|
||||
size_t end = text.size();
|
||||
while (start < end && (text[start] == ' ' || text[start] == '\t' || text[start] == '\n' || text[start] == '\r' || text[start] == '\x1e')) {
|
||||
start++;
|
||||
}
|
||||
while (end > start && (text[end - 1] == ' ' || text[end - 1] == '\t' || text[end - 1] == '\n' || text[end - 1] == '\r' || text[end - 1] == '\x1e')) {
|
||||
end--;
|
||||
}
|
||||
std::string cleaned(text.substr(start, end - start));
|
||||
if (format == simdjson::stream_format::comma_delimited || format == simdjson::stream_format::comma_delimited_array) {
|
||||
while (!cleaned.empty() && cleaned.back() == ',') {
|
||||
cleaned.pop_back();
|
||||
while (!cleaned.empty() && (cleaned.back() == ' ' || cleaned.back() == '\t' || cleaned.back() == '\n' || cleaned.back() == '\r')) {
|
||||
cleaned.pop_back();
|
||||
}
|
||||
}
|
||||
}
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
std::string canonicalize_document(simdjson::stream_format format, std::string_view text) {
|
||||
std::string cleaned = strip_stream_artifacts(format, text);
|
||||
simdjson::dom::parser parser;
|
||||
simdjson::dom::element doc;
|
||||
if (parser.parse(cleaned).get(doc)) {
|
||||
return std::string("PARSE_ERROR:") + cleaned;
|
||||
}
|
||||
return simdjson::minify(doc);
|
||||
}
|
||||
|
||||
const std::vector<std::string> &expected_documents() {
|
||||
static const std::vector<std::string> docs = document_stream_fuzz::make_documents();
|
||||
return docs;
|
||||
}
|
||||
|
||||
const std::vector<document_stream_fuzz::stream_case> &stream_cases() {
|
||||
static const std::vector<document_stream_fuzz::stream_case> cases =
|
||||
document_stream_fuzz::make_stream_cases(expected_documents());
|
||||
return cases;
|
||||
}
|
||||
|
||||
bool verify_case(const document_stream_fuzz::stream_case &test_case) {
|
||||
TEST_START();
|
||||
const auto &expected = test_case.expected_documents;
|
||||
ASSERT_TRUE(test_case.input.size() > document_stream_fuzz::batch_size * 4);
|
||||
|
||||
simdjson::padded_string input(test_case.input);
|
||||
simdjson::dom::parser parser;
|
||||
|
||||
for (int pass = 0; pass < 2; pass++) {
|
||||
simdjson::dom::document_stream stream;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, document_stream_fuzz::batch_size, test_case.format).get(stream));
|
||||
|
||||
size_t index = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(index < expected.size());
|
||||
simdjson::dom::element el;
|
||||
ASSERT_SUCCESS(doc.get(el));
|
||||
ASSERT_EQUAL(
|
||||
canonicalize_document(test_case.format, simdjson::minify(el)),
|
||||
canonicalize_document(test_case.format, expected[index])
|
||||
);
|
||||
index++;
|
||||
}
|
||||
ASSERT_EQUAL(index, expected.size());
|
||||
}
|
||||
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool run() {
|
||||
for (const auto &test_case : stream_cases()) {
|
||||
std::cout << "Running fuzz corpus against stream format: " << test_case.name << std::endl;
|
||||
if (!verify_case(test_case)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace document_stream_fuzz_tests
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
return test_main(argc, argv, document_stream_fuzz_tests::run);
|
||||
}
|
||||
@@ -1068,6 +1068,57 @@ namespace document_stream_tests {
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS-only input variants accept EMPTY or zero documents", ([&]() {
|
||||
// Several RS-only patterns. For each, parse_many must either:
|
||||
// (a) succeed and yield zero documents on iteration, or
|
||||
// (b) report simdjson::EMPTY.
|
||||
// Any other outcome (including a non-EMPTY error, or producing
|
||||
// ghost documents) is a failure.
|
||||
const std::string inputs[] = {
|
||||
"\x1e"s, // single RS, no LF
|
||||
"\x1e\n"s, // RS + LF
|
||||
"\x1e\x1e"s, // back-to-back RS
|
||||
"\x1e \t\n"s, // RS + whitespace
|
||||
"\x1e\n\x1e\n\x1e\n"s, // three RSes, no payloads
|
||||
};
|
||||
for (const auto& s : inputs) {
|
||||
auto input = simdjson::padded_string(s);
|
||||
auto result = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream);
|
||||
if (result == simdjson::SUCCESS) {
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { (void)doc; count++; }
|
||||
ASSERT_EQUAL(count, size_t(0));
|
||||
} else {
|
||||
ASSERT_EQUAL(result, simdjson::EMPTY);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("string documents current_index and source", ([&]() {
|
||||
// Layout: RS(0) "(1) h(2) e(3) l(4) l(5) o(6) "(7) LF(8)
|
||||
// RS(9) "(10) w(11) o(12) r(13) l(14) d(15) "(16) LF(17)
|
||||
// Regression test: when RS is followed by a JSON string, the
|
||||
// first structural after the RS is the opening quote. The stream
|
||||
// must report the document position at the opening quote (not the
|
||||
// RS) and source() must yield the quoted string verbatim.
|
||||
auto input = simdjson::padded_string("\x1e\"hello\"\n\x1e\"world\"\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"hello\""));
|
||||
std::string_view s1;
|
||||
ASSERT_SUCCESS((*it).get(s1));
|
||||
ASSERT_EQUAL(s1, std::string_view("hello"));
|
||||
++it;
|
||||
ASSERT_EQUAL(it.current_index(), size_t(10));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"world\""));
|
||||
std::string_view s2;
|
||||
ASSERT_SUCCESS((*it).get(s2));
|
||||
ASSERT_EQUAL(s2, std::string_view("world"));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("whitespace between RS and value", ([&]() {
|
||||
auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t\n{\"b\":2}\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
@@ -1129,6 +1180,443 @@ namespace document_stream_tests {
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar trio: number, true, string", ([&]() {
|
||||
// Layout: RS(0) 1(1) LF(2) RS(3) t(4) r(5) u(6) e(7) LF(8)
|
||||
// RS(9) "(10) x(11) "(12) LF(13)
|
||||
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
// doc 0: number 1
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("1"));
|
||||
int64_t i; ASSERT_SUCCESS((*it).get(i)); ASSERT_EQUAL(i, int64_t(1));
|
||||
++it;
|
||||
|
||||
// doc 1: true
|
||||
ASSERT_EQUAL(it.current_index(), size_t(4));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("true"));
|
||||
bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b);
|
||||
++it;
|
||||
|
||||
// doc 2: "x"
|
||||
ASSERT_EQUAL(it.current_index(), size_t(10));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"x\""));
|
||||
std::string_view s; ASSERT_SUCCESS((*it).get(s)); ASSERT_EQUAL(s, std::string_view("x"));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar trio multibatch (small batch_size)", ([&]() {
|
||||
// Same input as the scalar trio test, but batch_size=6 forces the
|
||||
// stream through multiple stage1 invocations. Behavior must match
|
||||
// the single-batch case exactly.
|
||||
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 6, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("1"));
|
||||
int64_t i; ASSERT_SUCCESS((*it).get(i)); ASSERT_EQUAL(i, int64_t(1));
|
||||
++it;
|
||||
|
||||
ASSERT_EQUAL(it.current_index(), size_t(4));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("true"));
|
||||
bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b);
|
||||
++it;
|
||||
|
||||
ASSERT_EQUAL(it.current_index(), size_t(10));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"x\""));
|
||||
std::string_view s; ASSERT_SUCCESS((*it).get(s)); ASSERT_EQUAL(s, std::string_view("x"));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch", ([&]() {
|
||||
// Stress the windowing / multi-stage1 path with 256 RS-prefixed
|
||||
// scalar documents rotating through all four scalar types. A small
|
||||
// batch_size (64) forces ~25 stage1 invocations over ~1.5 KB of
|
||||
// input. Every document must parse and appear in order.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
bytes += '\n';
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e));
|
||||
ASSERT_TRUE(e.is_null());
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() {
|
||||
// Same as the large stress test, but with extra RSes sprinkled every
|
||||
// 7 documents (empty leading record within a valid record sequence)
|
||||
// plus a trailing bare RS. Strict assertions: every document must
|
||||
// come through and parse cleanly, and the integer documents must
|
||||
// match the values we emitted.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
if (i % 7 == 0 && i > 0) { bytes += '\x1e'; } // empty record
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
bytes += '\n';
|
||||
}
|
||||
bytes += '\x1e'; // trailing bare RS
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e));
|
||||
ASSERT_TRUE(e.is_null());
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
// -------- RS spacing variants --------
|
||||
// The scanner classifies 0x1E as a scalar character, so the way an RS
|
||||
// is positioned relative to surrounding whitespace affects what
|
||||
// structural_indexes look like after stage1. These tests lock down
|
||||
// every reasonable arrangement: RS followed by the canonical LF, by
|
||||
// CRLF, by space, by tab, by mixed whitespace, with whitespace on both
|
||||
// sides, and tight (no surrounding whitespace at all) around
|
||||
// containers and strings.
|
||||
|
||||
SUBTEST("RS followed by LF then value (canonical RFC)", ([&]() {
|
||||
// bytes: 1e 31 0a 1e 32 0a 1e 33 0a
|
||||
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "2\n" "\x1e" "3\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {1, 4, 7};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by CRLF then value", ([&]() {
|
||||
// bytes: 1e 31 0d 0a 1e 32 0d 0a 1e 33 0d 0a
|
||||
auto input = simdjson::padded_string("\x1e" "1\r\n" "\x1e" "2\r\n" "\x1e" "3\r\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {1, 5, 9};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by single space then value", ([&]() {
|
||||
// bytes: 1e 20 31 0a 1e 20 32 0a 1e 20 33 0a
|
||||
auto input = simdjson::padded_string("\x1e 1\n\x1e 2\n\x1e 3\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {2, 6, 10}; // value sits after RS + 1 space
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by tab then value", ([&]() {
|
||||
// bytes: 1e 09 31 0a 1e 09 32 0a 1e 09 33 0a
|
||||
auto input = simdjson::padded_string("\x1e\t1\n\x1e\t2\n\x1e\t3\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {2, 6, 10}; // value sits after RS + 1 tab
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by mixed whitespace then value", ([&]() {
|
||||
// All four JSON whitespace characters between the RS and the value.
|
||||
// bytes: 1e 20 09 0d 0a 31 0a 1e 20 09 0d 0a 32 0a
|
||||
auto input = simdjson::padded_string("\x1e \t\r\n1\n\x1e \t\r\n2\n"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[2] = {1, 2};
|
||||
size_t expected_idx[2] = {5, 12}; // value sits after RS + 4 ws chars
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 2);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(2));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS with whitespace on both sides of scalar value", ([&]() {
|
||||
// Whitespace after the RS AND between the value and the next RS.
|
||||
// bytes: 1e 20 31 20 1e 20 32 20 1e 20 33
|
||||
auto input = simdjson::padded_string("\x1e 1 \x1e 2 \x1e 3"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {2, 6, 10};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS around arrays (no whitespace anywhere)", ([&]() {
|
||||
// No whitespace between RS and value, between values, or anywhere
|
||||
// else. Arrays survive because their closing ']' is a structural
|
||||
// character that naturally terminates the preceding scalar run.
|
||||
// bytes: 1e 5b 31 2c 32 5d 1e 5b 33 2c 34 5d 1e 5b 35 2c 36 5d
|
||||
auto input = simdjson::padded_string("\x1e[1,2]\x1e[3,4]\x1e[5,6]"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t expected_idx[3] = {1, 7, 13};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
simdjson::dom::array arr;
|
||||
ASSERT_SUCCESS(doc.get(arr));
|
||||
size_t n = 0;
|
||||
for (auto e : arr) { (void)e; n++; }
|
||||
ASSERT_EQUAL(n, size_t(2));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS around string scalars (no whitespace anywhere)", ([&]() {
|
||||
// bytes: 1e 22 61 22 1e 22 62 22 1e 22 63 22
|
||||
auto input = simdjson::padded_string("\x1e\"a\"\x1e\"b\"\x1e\"c\""s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
const char* expected[3] = {"a", "b", "c"};
|
||||
size_t expected_idx[3] = {1, 5, 9};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
std::string_view sv;
|
||||
ASSERT_SUCCESS(doc.get(sv));
|
||||
ASSERT_EQUAL(sv, std::string_view(expected[i]));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS around mixed containers (no whitespace anywhere)", ([&]() {
|
||||
// Object, array, object - all back-to-back with only RS between.
|
||||
// bytes: 1e 7b 22 61 22 3a 31 7d 1e 5b 31 2c 32 5d 1e 7b 22 62 22 3a 32 7d
|
||||
auto input = simdjson::padded_string("\x1e{\"a\":1}\x1e[1,2]\x1e{\"b\":2}"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
// doc 0: {"a":1}
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
int64_t a; ASSERT_SUCCESS((*it)["a"].get(a)); ASSERT_EQUAL(a, int64_t(1));
|
||||
++it;
|
||||
|
||||
// doc 1: [1,2]
|
||||
ASSERT_EQUAL(it.current_index(), size_t(9));
|
||||
simdjson::dom::array arr;
|
||||
ASSERT_SUCCESS((*it).get(arr));
|
||||
size_t arr_count = 0;
|
||||
for (auto e : arr) { (void)e; arr_count++; }
|
||||
ASSERT_EQUAL(arr_count, size_t(2));
|
||||
++it;
|
||||
|
||||
// doc 2: {"b":2}
|
||||
ASSERT_EQUAL(it.current_index(), size_t(15));
|
||||
int64_t b; ASSERT_SUCCESS((*it)["b"].get(b)); ASSERT_EQUAL(b, int64_t(2));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS multibatch: 100 objects no whitespace, bs=32", ([&]() {
|
||||
// 100 tight objects `\x1e{"i":0}\x1e{"i":1}...` with a small batch
|
||||
// size to stress the multibatch path. Objects are delimited by their
|
||||
// closing '}' so this case works without any whitespace.
|
||||
constexpr size_t N = 100;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
bytes += "{\"i\":";
|
||||
bytes += std::to_string(i);
|
||||
bytes += "}";
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 32, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc["i"].get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("spaced RS multibatch: 100 int scalars RS+space+value+LF, bs=32", ([&]() {
|
||||
// 100 int scalars shaped as `\x1e <n>\n` each. Small batch size
|
||||
// (32) forces ~16 stage1 invocations over the ~500 byte input.
|
||||
constexpr size_t N = 100;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
bytes += ' ';
|
||||
bytes += std::to_string(i);
|
||||
bytes += '\n';
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 32, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("leading/trailing/repeated RS separators around scalars", ([&]() {
|
||||
// RFC 7464 says each JSON text is preceded by exactly one RS. The
|
||||
// sensible interpretation of degenerate inputs:
|
||||
// - leading RS-RS (empty record before the first scalar): skip
|
||||
// - repeated RSes between two scalars (empty intermediate record):
|
||||
// skip the empty record, both real scalars are reported
|
||||
// - trailing bare RS with no payload: skip, no phantom document
|
||||
// Layout: RS(0) RS(1) 1(2) LF(3) RS(4) RS(5) RS(6) 2(7) LF(8) RS(9)
|
||||
auto input = simdjson::padded_string("\x1e\x1e" "1\n" "\x1e\x1e\x1e" "2\n" "\x1e"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[] = {1, 2};
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < 2);
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(2));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
@@ -1257,6 +1745,350 @@ namespace document_stream_tests {
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar quintet 1,2,\"x\",true,null", ([&]() {
|
||||
// Five comma-delimited scalars: number, number, string, true, null.
|
||||
// Layout: 1(0) ,(1) 2(2) ,(3) "(4) x(5) "(6) ,(7) t(8) r(9) u(10) e(11) ,(12) n(13) u(14) l(15) l(16)
|
||||
//
|
||||
// Note on source(): the comma_delimited filter strips root-level
|
||||
// commas from structural_indexes but the bytes remain in the buffer.
|
||||
// source() for a scalar slices [current_index, next_doc_index) and
|
||||
// strips trailing whitespace, so it currently includes the trailing
|
||||
// separator comma (e.g. doc 0 source = "1,"). This is the analogous
|
||||
// bug to the json_sequence RS-trailing source bug. The assertions
|
||||
// below lock down current_index() and parsed values strictly, and
|
||||
// accept source() either with or without a single trailing comma.
|
||||
auto input = R"(1,2,"x",true,null)"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
auto src_matches = [](std::string_view src, std::string_view want) {
|
||||
return src == want || src == std::string(want) + ",";
|
||||
};
|
||||
|
||||
// doc 0: 1
|
||||
ASSERT_EQUAL(it.current_index(), size_t(0));
|
||||
int64_t i1; ASSERT_SUCCESS((*it).get(i1)); ASSERT_EQUAL(i1, int64_t(1));
|
||||
ASSERT_TRUE(src_matches(it.source(), "1"));
|
||||
++it;
|
||||
|
||||
// doc 1: 2
|
||||
ASSERT_EQUAL(it.current_index(), size_t(2));
|
||||
int64_t i2; ASSERT_SUCCESS((*it).get(i2)); ASSERT_EQUAL(i2, int64_t(2));
|
||||
ASSERT_TRUE(src_matches(it.source(), "2"));
|
||||
++it;
|
||||
|
||||
// doc 2: "x"
|
||||
ASSERT_EQUAL(it.current_index(), size_t(4));
|
||||
std::string_view sv; ASSERT_SUCCESS((*it).get(sv)); ASSERT_EQUAL(sv, std::string_view("x"));
|
||||
ASSERT_TRUE(src_matches(it.source(), "\"x\""));
|
||||
++it;
|
||||
|
||||
// doc 3: true
|
||||
ASSERT_EQUAL(it.current_index(), size_t(8));
|
||||
bool b{}; ASSERT_SUCCESS((*it).get(b)); ASSERT_TRUE(b);
|
||||
ASSERT_TRUE(src_matches(it.source(), "true"));
|
||||
++it;
|
||||
|
||||
// doc 4: null (last doc, no trailing comma in source either way)
|
||||
ASSERT_EQUAL(it.current_index(), size_t(13));
|
||||
simdjson::dom::element e; ASSERT_SUCCESS((*it).get(e));
|
||||
ASSERT_TRUE(e.is_null());
|
||||
ASSERT_EQUAL(it.source(), std::string_view("null"));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar quintet multibatch (small batch_size)", ([&]() {
|
||||
// Same input as the scalar quintet test, but batch_size=8 forces the
|
||||
// stream through multiple stage1 invocations. Behavior must match
|
||||
// the single-batch case exactly.
|
||||
auto input = R"(1,2,"x",true,null)"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 8, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t count = 0;
|
||||
const size_t expected_idx[5] = {0, 2, 4, 8, 13};
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(count < 5);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[count]);
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(5));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch", ([&]() {
|
||||
// Stress the windowing / multi-stage1 path with 256 comma-delimited
|
||||
// scalar documents rotating through all four scalar types. A small
|
||||
// batch_size (64) forces ~20 stage1 invocations over ~1.3 KB of
|
||||
// input. Every document must parse and appear in order.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
if (i > 0) { bytes += ','; }
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e));
|
||||
ASSERT_TRUE(e.is_null());
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() {
|
||||
// Same as the large stress test, but with a leading comma, extra
|
||||
// commas sprinkled every 5 documents, and two trailing commas. The
|
||||
// comma_delimited filter is lenient about degenerate separators,
|
||||
// so this test is STRICT: exactly N documents must still come out,
|
||||
// in order, with correct values.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
bytes += ','; // leading comma
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
if (i > 0) { bytes += ','; }
|
||||
if (i % 5 == 0 && i > 0) { bytes += ','; } // extra comma
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
}
|
||||
bytes += ",,"; // trailing commas
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
simdjson::dom::element e; ASSERT_SUCCESS(doc.get(e));
|
||||
ASSERT_TRUE(e.is_null());
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("leading/trailing/repeated commas around scalars", ([&]() {
|
||||
// Comma-delimited filter is expected to be lenient: leading commas,
|
||||
// trailing commas, and repeated commas between scalars all collapse
|
||||
// to single separators. The three real scalars (1, 2, "x") must be
|
||||
// reported with no phantom documents.
|
||||
// Layout: ,(0) 1(1) ,(2) ,(3) 2(4) ,(5) ,(6) "(7) x(8) "(9) ,(10) ,(11)
|
||||
auto input = R"(,1,,2,,"x",,)"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t count = 0;
|
||||
int64_t i1 = -1, i2 = -1;
|
||||
std::string_view sx;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
if (count == 0) { ASSERT_SUCCESS(doc.get(i1)); }
|
||||
else if (count == 1) { ASSERT_SUCCESS(doc.get(i2)); }
|
||||
else if (count == 2) { ASSERT_SUCCESS(doc.get(sx)); }
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(3));
|
||||
ASSERT_EQUAL(i1, int64_t(1));
|
||||
ASSERT_EQUAL(i2, int64_t(2));
|
||||
ASSERT_EQUAL(sx, std::string_view("x"));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
// -------- stream_format::comma_delimited_array --------
|
||||
// The comma_delimited_array mode strips the outer [ ] (plus any
|
||||
// surrounding JSON whitespace) and then delegates to comma_delimited.
|
||||
|
||||
SUBTEST("comma_delimited_array basic [1,2,3]", ([&]() {
|
||||
auto input = R"([1,2,3])"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
int64_t got[3] = {0, 0, 0};
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(count < 3);
|
||||
ASSERT_SUCCESS(doc.get(got[count]));
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(3));
|
||||
ASSERT_EQUAL(got[0], int64_t(1));
|
||||
ASSERT_EQUAL(got[1], int64_t(2));
|
||||
ASSERT_EQUAL(got[2], int64_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array mixed scalar types", ([&]() {
|
||||
auto input = R"([1, "a", true, null, {"x":1}])"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
|
||||
ASSERT_EQUAL(count, size_t(5));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array empty []", ([&]() {
|
||||
auto input = R"([])"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { (void)doc; count++; }
|
||||
ASSERT_EQUAL(count, size_t(0));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array empty with interior whitespace [ ]", ([&]() {
|
||||
// [ ] should also yield zero documents.
|
||||
auto input = simdjson::padded_string("[ \t\n]"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { (void)doc; count++; }
|
||||
ASSERT_EQUAL(count, size_t(0));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array surrounding whitespace stripped", ([&]() {
|
||||
// All four JSON whitespace characters before the '[' and after the ']'.
|
||||
auto input = simdjson::padded_string(" \t\n\r[ 1, 2, 3 ] \t\n\r"s);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
int64_t sum = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc.get(v));
|
||||
sum += v;
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(3));
|
||||
ASSERT_EQUAL(sum, int64_t(6));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array nested commas preserved", ([&]() {
|
||||
auto input = R"([{"a":1,"b":2},{"c":[3,4,5]}])"_padded;
|
||||
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
auto it = stream.begin();
|
||||
// doc 0: {"a":1,"b":2}
|
||||
int64_t a, b;
|
||||
ASSERT_SUCCESS((*it)["a"].get(a));
|
||||
ASSERT_SUCCESS((*it)["b"].get(b));
|
||||
ASSERT_EQUAL(a, int64_t(1));
|
||||
ASSERT_EQUAL(b, int64_t(2));
|
||||
++it;
|
||||
// doc 1: {"c":[3,4,5]}
|
||||
simdjson::dom::array arr;
|
||||
ASSERT_SUCCESS((*it)["c"].get(arr));
|
||||
size_t arr_count = 0;
|
||||
for (auto e : arr) { (void)e; arr_count++; }
|
||||
ASSERT_EQUAL(arr_count, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array missing leading bracket is TAPE_ERROR", ([&]() {
|
||||
auto input = R"(1,2,3)"_padded;
|
||||
auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array missing trailing bracket is TAPE_ERROR", ([&]() {
|
||||
auto input = R"([1,2,3)"_padded;
|
||||
auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array empty input is TAPE_ERROR", ([&]() {
|
||||
auto input = simdjson::padded_string(""s);
|
||||
auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array whitespace-only input is TAPE_ERROR", ([&]() {
|
||||
auto input = simdjson::padded_string(" \t\n"s);
|
||||
auto err = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array multibatch large array", ([&]() {
|
||||
// Generate [0,1,2,...,N-1] and iterate with a small batch_size so the
|
||||
// stream runs stage1 across many windows. All N values must come out
|
||||
// in order with no phantom documents.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes = "[";
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
if (i > 0) { bytes += ','; }
|
||||
bytes += std::to_string(i);
|
||||
}
|
||||
bytes += "]";
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.parse_many(input, 64, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) {
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(count < N);
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc.get(v));
|
||||
ASSERT_EQUAL(v, int64_t(count));
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ add_cpp_test(ondemand_array_tests LABELS ondemand acceptance
|
||||
add_cpp_test(ondemand_array_error_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_compilation_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_document_stream_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_document_stream_fuzz_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_error_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_error_location_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_json_pointer_tests LABELS ondemand acceptance per_implementation)
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "simdjson.h"
|
||||
#include "test_ondemand.h"
|
||||
#include "document_stream_fuzz_test_common.h"
|
||||
|
||||
namespace document_stream_fuzz_tests {
|
||||
|
||||
std::string strip_stream_artifacts(simdjson::stream_format format, std::string_view text) {
|
||||
size_t start = 0;
|
||||
size_t end = text.size();
|
||||
while (start < end && (text[start] == ' ' || text[start] == '\t' || text[start] == '\n' || text[start] == '\r' || text[start] == '\x1e')) {
|
||||
start++;
|
||||
}
|
||||
while (end > start && (text[end - 1] == ' ' || text[end - 1] == '\t' || text[end - 1] == '\n' || text[end - 1] == '\r' || text[end - 1] == '\x1e')) {
|
||||
end--;
|
||||
}
|
||||
std::string cleaned(text.substr(start, end - start));
|
||||
if (format == simdjson::stream_format::comma_delimited || format == simdjson::stream_format::comma_delimited_array) {
|
||||
while (!cleaned.empty() && cleaned.back() == ',') {
|
||||
cleaned.pop_back();
|
||||
while (!cleaned.empty() && (cleaned.back() == ' ' || cleaned.back() == '\t' || cleaned.back() == '\n' || cleaned.back() == '\r')) {
|
||||
cleaned.pop_back();
|
||||
}
|
||||
}
|
||||
}
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
std::string canonicalize_document(simdjson::stream_format format, std::string_view text) {
|
||||
std::string cleaned = strip_stream_artifacts(format, text);
|
||||
simdjson::dom::parser parser;
|
||||
simdjson::dom::element doc;
|
||||
if (parser.parse(cleaned).get(doc)) {
|
||||
return std::string("PARSE_ERROR:") + cleaned;
|
||||
}
|
||||
return simdjson::minify(doc);
|
||||
}
|
||||
|
||||
const std::vector<std::string> &expected_documents() {
|
||||
static const std::vector<std::string> docs = document_stream_fuzz::make_documents();
|
||||
return docs;
|
||||
}
|
||||
|
||||
const std::vector<document_stream_fuzz::stream_case> &stream_cases() {
|
||||
static const std::vector<document_stream_fuzz::stream_case> cases =
|
||||
document_stream_fuzz::make_stream_cases(expected_documents());
|
||||
return cases;
|
||||
}
|
||||
|
||||
bool verify_case(const document_stream_fuzz::stream_case &test_case) {
|
||||
TEST_START();
|
||||
const auto &expected = test_case.expected_documents;
|
||||
ASSERT_TRUE(test_case.input.size() > document_stream_fuzz::batch_size * 4);
|
||||
|
||||
simdjson::padded_string input(test_case.input);
|
||||
simdjson::ondemand::parser parser;
|
||||
|
||||
for (int pass = 0; pass < 2; pass++) {
|
||||
simdjson::ondemand::document_stream stream;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, document_stream_fuzz::batch_size, test_case.format).get(stream));
|
||||
|
||||
size_t index = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_result = *it;
|
||||
ASSERT_SUCCESS(doc_result.error());
|
||||
ASSERT_TRUE(index < expected.size());
|
||||
|
||||
simdjson::ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_result.get(doc));
|
||||
|
||||
std::string_view actual;
|
||||
ASSERT_SUCCESS(simdjson::to_json_string(doc).get(actual));
|
||||
ASSERT_EQUAL(
|
||||
canonicalize_document(test_case.format, actual),
|
||||
canonicalize_document(test_case.format, expected[index])
|
||||
);
|
||||
index++;
|
||||
}
|
||||
ASSERT_EQUAL(index, expected.size());
|
||||
}
|
||||
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
bool run() {
|
||||
for (const auto &test_case : stream_cases()) {
|
||||
std::cout << "Running fuzz corpus against stream format: " << test_case.name << std::endl;
|
||||
if (!verify_case(test_case)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace document_stream_fuzz_tests
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
return test_main(argc, argv, document_stream_fuzz_tests::run);
|
||||
}
|
||||
@@ -1031,6 +1031,65 @@ namespace document_stream_tests {
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS-only input variants accept EMPTY or zero documents", ([&]() {
|
||||
// Several RS-only patterns. For each, iterate_many must either:
|
||||
// (a) succeed and yield zero documents on iteration, or
|
||||
// (b) report simdjson::EMPTY.
|
||||
// Any other outcome (including a non-EMPTY error, or producing
|
||||
// ghost documents) is a failure.
|
||||
const std::string inputs[] = {
|
||||
"\x1e"s, // single RS, no LF
|
||||
"\x1e\n"s, // RS + LF
|
||||
"\x1e\x1e"s, // back-to-back RS
|
||||
"\x1e \t\n"s, // RS + whitespace
|
||||
"\x1e\n\x1e\n\x1e\n"s, // three RSes, no payloads
|
||||
};
|
||||
for (const auto& s : inputs) {
|
||||
auto input = simdjson::padded_string(s);
|
||||
auto result = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream);
|
||||
if (result == simdjson::SUCCESS) {
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { (void)doc; count++; }
|
||||
ASSERT_EQUAL(count, size_t(0));
|
||||
} else {
|
||||
ASSERT_EQUAL(result, simdjson::EMPTY);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("string documents current_index and source", ([&]() {
|
||||
// Layout: RS(0) "(1) h(2) e(3) l(4) l(5) o(6) "(7) LF(8)
|
||||
// RS(9) "(10) w(11) o(12) r(13) l(14) d(15) "(16) LF(17)
|
||||
// Regression test: when RS is followed by a JSON string, the
|
||||
// first structural after the RS is the opening quote. The stream
|
||||
// must report the document position at the opening quote (not the
|
||||
// RS) and source() must yield the quoted string verbatim.
|
||||
auto input = simdjson::padded_string("\x1e\"hello\"\n\x1e\"world\"\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"hello\""));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
std::string_view sv;
|
||||
ASSERT_SUCCESS(doc.get_string().get(sv));
|
||||
ASSERT_EQUAL(sv, std::string_view("hello"));
|
||||
}
|
||||
++it;
|
||||
ASSERT_EQUAL(it.current_index(), size_t(10));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"world\""));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
std::string_view sv;
|
||||
ASSERT_SUCCESS(doc.get_string().get(sv));
|
||||
ASSERT_EQUAL(sv, std::string_view("world"));
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("whitespace between RS and value", ([&]() {
|
||||
auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t\n{\"b\":2}\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
@@ -1110,6 +1169,503 @@ namespace document_stream_tests {
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar trio: number, true, string", ([&]() {
|
||||
// Layout: RS(0) 1(1) LF(2) RS(3) t(4) r(5) u(6) e(7) LF(8)
|
||||
// RS(9) "(10) x(11) "(12) LF(13)
|
||||
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
// doc 0: number 1
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("1"));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(1));
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 1: true
|
||||
ASSERT_EQUAL(it.current_index(), size_t(4));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("true"));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
bool v; ASSERT_SUCCESS(doc.get_bool().get(v));
|
||||
ASSERT_TRUE(v);
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 2: "x"
|
||||
ASSERT_EQUAL(it.current_index(), size_t(10));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"x\""));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
std::string_view sv; ASSERT_SUCCESS(doc.get_string().get(sv));
|
||||
ASSERT_EQUAL(sv, std::string_view("x"));
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar trio multibatch (small batch_size)", ([&]() {
|
||||
// Same input as the scalar trio test, but batch_size=6 forces
|
||||
// the stream through multiple stage1 invocations. Behavior must
|
||||
// match the single-batch case for current_index and source.
|
||||
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "true\n" "\x1e" "\"x\"\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 6, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("1"));
|
||||
++it;
|
||||
ASSERT_EQUAL(it.current_index(), size_t(4));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("true"));
|
||||
++it;
|
||||
ASSERT_EQUAL(it.current_index(), size_t(10));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("\"x\""));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch", ([&]() {
|
||||
// Stress the windowing / multi-stage1 path with 256 RS-prefixed
|
||||
// scalar documents rotating through all four scalar types. A
|
||||
// small batch_size (64) forces ~25 stage1 invocations over
|
||||
// ~1.5 KB of input. Every document must parse and appear in
|
||||
// order.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
bytes += '\n';
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get_bool().get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
bool is_null;
|
||||
ASSERT_SUCCESS(doc.is_null().get(is_null));
|
||||
ASSERT_TRUE(is_null);
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() {
|
||||
// Same as the large stress test, but with extra RSes sprinkled
|
||||
// every 7 documents (empty leading record within a valid
|
||||
// record sequence) plus a trailing bare RS. Strict
|
||||
// assertions: every document must come through and parse
|
||||
// cleanly, with the correct typed value.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
if (i % 7 == 0 && i > 0) { bytes += '\x1e'; } // empty record
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
bytes += '\n';
|
||||
}
|
||||
bytes += '\x1e'; // trailing bare RS
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get_bool().get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
bool is_null;
|
||||
ASSERT_SUCCESS(doc.is_null().get(is_null));
|
||||
ASSERT_TRUE(is_null);
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
// -------- RS spacing variants --------
|
||||
// The scanner classifies 0x1E as a scalar character, so the way an
|
||||
// RS is positioned relative to surrounding whitespace affects what
|
||||
// structural_indexes look like after stage1. These tests lock down
|
||||
// every reasonable arrangement: RS followed by the canonical LF,
|
||||
// by CRLF, by space, by tab, by mixed whitespace, with whitespace
|
||||
// on both sides, and tight (no surrounding whitespace at all)
|
||||
// around containers and strings.
|
||||
|
||||
SUBTEST("RS followed by LF then value (canonical RFC)", ([&]() {
|
||||
// bytes: 1e 31 0a 1e 32 0a 1e 33 0a
|
||||
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "2\n" "\x1e" "3\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {1, 4, 7};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by CRLF then value", ([&]() {
|
||||
// bytes: 1e 31 0d 0a 1e 32 0d 0a 1e 33 0d 0a
|
||||
auto input = simdjson::padded_string("\x1e" "1\r\n" "\x1e" "2\r\n" "\x1e" "3\r\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {1, 5, 9};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by single space then value", ([&]() {
|
||||
// bytes: 1e 20 31 0a 1e 20 32 0a 1e 20 33 0a
|
||||
auto input = simdjson::padded_string("\x1e 1\n\x1e 2\n\x1e 3\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {2, 6, 10};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by tab then value", ([&]() {
|
||||
// bytes: 1e 09 31 0a 1e 09 32 0a 1e 09 33 0a
|
||||
auto input = simdjson::padded_string("\x1e\t1\n\x1e\t2\n\x1e\t3\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {2, 6, 10};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS followed by mixed whitespace then value", ([&]() {
|
||||
// All four JSON whitespace characters between the RS and the
|
||||
// value. bytes: 1e 20 09 0d 0a 31 0a 1e 20 09 0d 0a 32 0a
|
||||
auto input = simdjson::padded_string("\x1e \t\r\n1\n\x1e \t\r\n2\n"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[2] = {1, 2};
|
||||
size_t expected_idx[2] = {5, 12};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 2);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(2));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("RS with whitespace on both sides of scalar value", ([&]() {
|
||||
// Whitespace after the RS AND between the value and the next RS.
|
||||
// bytes: 1e 20 31 20 1e 20 32 20 1e 20 33
|
||||
auto input = simdjson::padded_string("\x1e 1 \x1e 2 \x1e 3"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[3] = {1, 2, 3};
|
||||
size_t expected_idx[3] = {2, 6, 10};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS around arrays (no whitespace anywhere)", ([&]() {
|
||||
// No whitespace between RS and value, between values, or
|
||||
// anywhere else. Arrays survive because their closing ']' is a
|
||||
// structural character that naturally terminates the preceding
|
||||
// scalar run.
|
||||
// bytes: 1e 5b 31 2c 32 5d 1e 5b 33 2c 34 5d 1e 5b 35 2c 36 5d
|
||||
auto input = simdjson::padded_string("\x1e[1,2]\x1e[3,4]\x1e[5,6]"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t expected_idx[3] = {1, 7, 13};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
ondemand::array arr;
|
||||
ASSERT_SUCCESS(doc.get_array().get(arr));
|
||||
size_t n = 0;
|
||||
for (auto e : arr) { (void)e; n++; }
|
||||
ASSERT_EQUAL(n, size_t(2));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS around string scalars (no whitespace anywhere)", ([&]() {
|
||||
// bytes: 1e 22 61 22 1e 22 62 22 1e 22 63 22
|
||||
auto input = simdjson::padded_string("\x1e\"a\"\x1e\"b\"\x1e\"c\""s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
const char* expected[3] = {"a", "b", "c"};
|
||||
size_t expected_idx[3] = {1, 5, 9};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 3);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[i]);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
std::string_view sv;
|
||||
ASSERT_SUCCESS(doc.get_string().get(sv));
|
||||
ASSERT_EQUAL(sv, std::string_view(expected[i]));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS around mixed containers (no whitespace anywhere)", ([&]() {
|
||||
// Object, array, object - all back-to-back with only RS between.
|
||||
auto input = simdjson::padded_string("\x1e{\"a\":1}\x1e[1,2]\x1e{\"b\":2}"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
// doc 0: {"a":1}
|
||||
ASSERT_EQUAL(it.current_index(), size_t(1));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
int64_t a; ASSERT_SUCCESS(doc["a"].get(a));
|
||||
ASSERT_EQUAL(a, int64_t(1));
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 1: [1,2]
|
||||
ASSERT_EQUAL(it.current_index(), size_t(9));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
ondemand::array arr;
|
||||
ASSERT_SUCCESS(doc.get_array().get(arr));
|
||||
size_t arr_count = 0;
|
||||
for (auto e : arr) { (void)e; arr_count++; }
|
||||
ASSERT_EQUAL(arr_count, size_t(2));
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 2: {"b":2}
|
||||
ASSERT_EQUAL(it.current_index(), size_t(15));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
int64_t b; ASSERT_SUCCESS(doc["b"].get(b));
|
||||
ASSERT_EQUAL(b, int64_t(2));
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("tight RS multibatch: 100 objects no whitespace, bs=32", ([&]() {
|
||||
// 100 tight objects `\x1e{"i":0}\x1e{"i":1}...` with a small
|
||||
// batch size to stress the multibatch path. Objects are
|
||||
// delimited by their closing '}' so this case works without
|
||||
// any whitespace.
|
||||
constexpr size_t N = 100;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
bytes += "{\"i\":";
|
||||
bytes += std::to_string(i);
|
||||
bytes += "}";
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 32, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc["i"].get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("spaced RS multibatch: 100 int scalars RS+space+value+LF, bs=32", ([&]() {
|
||||
// 100 int scalars shaped as `\x1e <n>\n` each. Small batch
|
||||
// size (32) forces ~16 stage1 invocations over the ~500 byte
|
||||
// input.
|
||||
constexpr size_t N = 100;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
bytes += '\x1e';
|
||||
bytes += ' ';
|
||||
bytes += std::to_string(i);
|
||||
bytes += '\n';
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 32, simdjson::stream_format::json_sequence).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("leading/trailing/repeated RS separators around scalars", ([&]() {
|
||||
// RFC 7464 says each JSON text is preceded by exactly one RS.
|
||||
// The sensible interpretation of degenerate inputs:
|
||||
// - leading RS-RS (empty record before the first scalar): skip
|
||||
// - repeated RSes between two scalars (empty intermediate
|
||||
// record): skip the empty record, both scalars reported
|
||||
// - trailing bare RS with no payload: skip, no phantom doc
|
||||
// Layout: RS(0) RS(1) 1(2) LF(3) RS(4) RS(5) RS(6) 2(7) LF(8) RS(9)
|
||||
auto input = simdjson::padded_string("\x1e\x1e" "1\n" "\x1e\x1e\x1e" "2\n" "\x1e"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
|
||||
int64_t expected[] = {1, 2};
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < 2);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, expected[i]);
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, size_t(2));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
@@ -1247,6 +1803,405 @@ namespace document_stream_tests {
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar quintet 1,2,\"x\",true,null", ([&]() {
|
||||
// Five comma-delimited scalars: number, number, string, true, null.
|
||||
// Layout: 1(0) ,(1) 2(2) ,(3) "(4) x(5) "(6) ,(7) t(8) r(9) u(10) e(11) ,(12) n(13) u(14) l(15) l(16)
|
||||
//
|
||||
// Note on source(): the comma_delimited filter strips root-level
|
||||
// commas from structural_indexes but the bytes remain in the
|
||||
// buffer. source() for a scalar slices [current_index,
|
||||
// next_doc_index) and strips trailing whitespace, so it
|
||||
// currently includes the trailing separator comma (e.g. doc 0
|
||||
// source = "1,"). The assertions below lock down current_index()
|
||||
// and parsed values strictly, and accept source() either with
|
||||
// or without a single trailing comma.
|
||||
auto input = R"(1,2,"x",true,null)"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
|
||||
auto it = stream.begin();
|
||||
|
||||
auto src_matches = [](std::string_view src, std::string_view want) {
|
||||
return src == want || src == std::string(want) + ",";
|
||||
};
|
||||
|
||||
// doc 0: 1
|
||||
ASSERT_EQUAL(it.current_index(), size_t(0));
|
||||
ASSERT_TRUE(src_matches(it.source(), "1"));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(1));
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 1: 2
|
||||
ASSERT_EQUAL(it.current_index(), size_t(2));
|
||||
ASSERT_TRUE(src_matches(it.source(), "2"));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(2));
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 2: "x"
|
||||
ASSERT_EQUAL(it.current_index(), size_t(4));
|
||||
ASSERT_TRUE(src_matches(it.source(), "\"x\""));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
std::string_view sv; ASSERT_SUCCESS(doc.get_string().get(sv));
|
||||
ASSERT_EQUAL(sv, std::string_view("x"));
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 3: true
|
||||
ASSERT_EQUAL(it.current_index(), size_t(8));
|
||||
ASSERT_TRUE(src_matches(it.source(), "true"));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
bool v; ASSERT_SUCCESS(doc.get_bool().get(v));
|
||||
ASSERT_TRUE(v);
|
||||
}
|
||||
++it;
|
||||
|
||||
// doc 4: null (last doc, no trailing comma in source either way)
|
||||
ASSERT_EQUAL(it.current_index(), size_t(13));
|
||||
ASSERT_EQUAL(it.source(), std::string_view("null"));
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
bool is_null;
|
||||
ASSERT_SUCCESS(doc.is_null().get(is_null));
|
||||
ASSERT_TRUE(is_null);
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("scalar quintet multibatch (small batch_size)", ([&]() {
|
||||
// Same input as the scalar quintet test, but batch_size=8 forces
|
||||
// the stream through multiple stage1 invocations. Behavior must
|
||||
// match the single-batch case for current_index.
|
||||
auto input = R"(1,2,"x",true,null)"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 8, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t count = 0;
|
||||
const size_t expected_idx[5] = {0, 2, 4, 8, 13};
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc = *it;
|
||||
ASSERT_SUCCESS(doc.error());
|
||||
ASSERT_TRUE(count < 5);
|
||||
ASSERT_EQUAL(it.current_index(), expected_idx[count]);
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(5));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch", ([&]() {
|
||||
// Stress the windowing / multi-stage1 path with 256 comma-
|
||||
// delimited scalar documents rotating through all four scalar
|
||||
// types. A small batch_size (64) forces ~20 stage1 invocations
|
||||
// over ~1.3 KB of input. Every document must parse and appear
|
||||
// in order.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
if (i > 0) { bytes += ','; }
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
}
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get_bool().get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
bool is_null;
|
||||
ASSERT_SUCCESS(doc.is_null().get(is_null));
|
||||
ASSERT_TRUE(is_null);
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("large synthetic scalar stream multibatch with separator noise", ([&]() {
|
||||
// Same as the large stress test, but with a leading comma,
|
||||
// extra commas sprinkled every 5 documents, and two trailing
|
||||
// commas. The comma_delimited filter is lenient about
|
||||
// degenerate separators, so this test is STRICT: exactly N
|
||||
// documents must still come out, in order, with correct
|
||||
// values.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes;
|
||||
bytes += ','; // leading comma
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
if (i > 0) { bytes += ','; }
|
||||
if (i % 5 == 0 && i > 0) { bytes += ','; } // extra comma
|
||||
switch (i % 4) {
|
||||
case 0: bytes += std::to_string(i); break;
|
||||
case 1: bytes += (i & 1) ? "true" : "false"; break;
|
||||
case 2: bytes += "\"s" + std::to_string(i) + "\""; break;
|
||||
case 3: bytes += "null"; break;
|
||||
}
|
||||
}
|
||||
bytes += ",,"; // trailing commas
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t i = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(i < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
switch (i % 4) {
|
||||
case 0: {
|
||||
int64_t v; ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(i));
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
bool v; ASSERT_SUCCESS(doc.get_bool().get(v));
|
||||
ASSERT_EQUAL(v, bool(i & 1));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::string_view v; ASSERT_SUCCESS(doc.get_string().get(v));
|
||||
std::string want = "s" + std::to_string(i);
|
||||
ASSERT_EQUAL(v, std::string_view(want));
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
bool is_null;
|
||||
ASSERT_SUCCESS(doc.is_null().get(is_null));
|
||||
ASSERT_TRUE(is_null);
|
||||
break;
|
||||
}
|
||||
}
|
||||
i++;
|
||||
}
|
||||
ASSERT_EQUAL(i, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("leading/trailing/repeated commas around scalars", ([&]() {
|
||||
// Comma-delimited filter is expected to be lenient: leading
|
||||
// commas, trailing commas, and repeated commas between scalars
|
||||
// all collapse to single separators. The three real scalars
|
||||
// (1, 2, "x") must be reported with no phantom documents.
|
||||
// Layout: ,(0) 1(1) ,(2) ,(3) 2(4) ,(5) ,(6) "(7) x(8) "(9) ,(10) ,(11)
|
||||
auto input = R"(,1,,2,,"x",,)"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
|
||||
size_t count = 0;
|
||||
int64_t i1 = -1, i2 = -1;
|
||||
std::string_view sx;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
if (count == 0) { ASSERT_SUCCESS(doc.get_int64().get(i1)); }
|
||||
else if (count == 1) { ASSERT_SUCCESS(doc.get_int64().get(i2)); }
|
||||
else if (count == 2) { ASSERT_SUCCESS(doc.get_string().get(sx)); }
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(3));
|
||||
ASSERT_EQUAL(i1, int64_t(1));
|
||||
ASSERT_EQUAL(i2, int64_t(2));
|
||||
ASSERT_EQUAL(sx, std::string_view("x"));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
// -------- stream_format::comma_delimited_array --------
|
||||
// The comma_delimited_array mode strips the outer [ ] (plus any
|
||||
// surrounding JSON whitespace) and then delegates to comma_delimited.
|
||||
|
||||
SUBTEST("comma_delimited_array basic [1,2,3]", ([&]() {
|
||||
auto input = R"([1,2,3])"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
int64_t got[3] = {0, 0, 0};
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(count < 3);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
ASSERT_SUCCESS(doc.get_int64().get(got[count]));
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(3));
|
||||
ASSERT_EQUAL(got[0], int64_t(1));
|
||||
ASSERT_EQUAL(got[1], int64_t(2));
|
||||
ASSERT_EQUAL(got[2], int64_t(3));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array mixed scalar types", ([&]() {
|
||||
auto input = R"([1, "a", true, null, {"x":1}])"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
|
||||
ASSERT_EQUAL(count, size_t(5));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array empty []", ([&]() {
|
||||
auto input = R"([])"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { (void)doc; count++; }
|
||||
ASSERT_EQUAL(count, size_t(0));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array empty with interior whitespace [ ]", ([&]() {
|
||||
auto input = simdjson::padded_string("[ \t\n]"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto doc : stream) { (void)doc; count++; }
|
||||
ASSERT_EQUAL(count, size_t(0));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array surrounding whitespace stripped", ([&]() {
|
||||
// All four JSON whitespace characters before the '[' and after
|
||||
// the ']'.
|
||||
auto input = simdjson::padded_string(" \t\n\r[ 1, 2, 3 ] \t\n\r"s);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
int64_t sum = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
sum += v;
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, size_t(3));
|
||||
ASSERT_EQUAL(sum, int64_t(6));
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array nested commas preserved", ([&]() {
|
||||
auto input = R"([{"a":1,"b":2},{"c":[3,4,5]}])"_padded;
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
auto it = stream.begin();
|
||||
// doc 0: {"a":1,"b":2}
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
int64_t a, b;
|
||||
ASSERT_SUCCESS(doc["a"].get(a));
|
||||
ASSERT_SUCCESS(doc["b"].get(b));
|
||||
ASSERT_EQUAL(a, int64_t(1));
|
||||
ASSERT_EQUAL(b, int64_t(2));
|
||||
}
|
||||
++it;
|
||||
// doc 1: {"c":[3,4,5]}
|
||||
{
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS((*it).get(doc));
|
||||
ondemand::array arr;
|
||||
ASSERT_SUCCESS(doc["c"].get_array().get(arr));
|
||||
size_t arr_count = 0;
|
||||
for (auto e : arr) { (void)e; arr_count++; }
|
||||
ASSERT_EQUAL(arr_count, size_t(3));
|
||||
}
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array missing leading bracket is TAPE_ERROR", ([&]() {
|
||||
auto input = R"(1,2,3)"_padded;
|
||||
auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array missing trailing bracket is TAPE_ERROR", ([&]() {
|
||||
auto input = R"([1,2,3)"_padded;
|
||||
auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array empty input is TAPE_ERROR", ([&]() {
|
||||
auto input = simdjson::padded_string(""s);
|
||||
auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array whitespace-only input is TAPE_ERROR", ([&]() {
|
||||
auto input = simdjson::padded_string(" \t\n"s);
|
||||
auto err = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited_array).get(stream);
|
||||
ASSERT_ERROR(err, simdjson::TAPE_ERROR);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
SUBTEST("comma_delimited_array multibatch large array", ([&]() {
|
||||
// Generate [0,1,2,...,N-1] and iterate with a small batch_size
|
||||
// so the stream runs stage1 across many windows. All N values
|
||||
// must come out in order with no phantom documents.
|
||||
constexpr size_t N = 256;
|
||||
std::string bytes = "[";
|
||||
for (size_t i = 0; i < N; i++) {
|
||||
if (i > 0) { bytes += ','; }
|
||||
bytes += std::to_string(i);
|
||||
}
|
||||
bytes += "]";
|
||||
auto input = simdjson::padded_string(bytes);
|
||||
ASSERT_SUCCESS(parser.iterate_many(input, 64, simdjson::stream_format::comma_delimited_array).get(stream));
|
||||
size_t count = 0;
|
||||
for (auto it = stream.begin(); it != stream.end(); ++it) {
|
||||
auto doc_res = *it;
|
||||
ASSERT_SUCCESS(doc_res.error());
|
||||
ASSERT_TRUE(count < N);
|
||||
ondemand::document_reference doc;
|
||||
ASSERT_SUCCESS(doc_res.get(doc));
|
||||
int64_t v;
|
||||
ASSERT_SUCCESS(doc.get_int64().get(v));
|
||||
ASSERT_EQUAL(v, int64_t(count));
|
||||
count++;
|
||||
}
|
||||
ASSERT_EQUAL(count, N);
|
||||
return true;
|
||||
}()));
|
||||
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user