admin/simdjson-simdjson
1
0
Fork 0
mirror of https://github.com/simdjson/simdjson synced 2026-06-08 17:27:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for RFC 7464 JSON text sequences and comma-delimited documents (#2664)

Browse Source 
* add support for RFC 7464 documents

* add threaded comma-delimited parse_many support

* fix failing tests in CI
This commit is contained in:
Jaël Champagne Gareau
2026-04-10 20:26:00 -04:00
committed by GitHub
parent 1a57afec1f
commit 72e51a9a81
21 changed files with 1549 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/CMakeLists.txt
+1
View File
@@ -15,6 +15,7 @@ if (TARGET benchmark::benchmark)
link_libraries(benchmark::benchmark)
add_executable(bench_parse_call bench_parse_call.cpp)
add_executable(bench_dom_api bench_dom_api.cpp)
add_executable(bench_stream_formats bench_stream_formats.cpp)
if(SIMDJSON_EXCEPTIONS)
add_executable(bench_ondemand bench_ondemand.cpp)
if(TARGET yyjson)
benchmark/bench_stream_formats.cpp
+216
View File
@@ -0,0 +1,216 @@
#include <benchmark/benchmark.h>
#include <string>
#include "simdjson.h"
using namespace simdjson;
namespace {
enum class stream_case {
ndjson_small,
ndjson_large,
rfc7464_small,
rfc7464_large,
comma_delimited_small,
comma_delimited_large
};
constexpr size_t TARGET_BYTES = 128 * 1000 * 1000;
constexpr size_t SMALL_PAYLOAD = 16;
constexpr size_t LARGE_PAYLOAD = 4096;
constexpr size_t BATCH_SIZE = 1 << 20;
struct stream_dataset {
padded_string json;
size_t count{};
};
std::string make_document(size_t id, size_t payload_size) {
return std::string{"{\"id\":"} + std::to_string(id) +
",\"name\":\"aaaaaaaa\",\"payload\":\"" +
std::string(payload_size, 'x') + "\",\"flag\":true}";
}
stream_dataset build_dataset(stream_case which) {
const bool small = which == stream_case::ndjson_small ||
which == stream_case::rfc7464_small ||
which == stream_case::comma_delimited_small;
const bool rfc = which == stream_case::rfc7464_small ||
which == stream_case::rfc7464_large;
const bool comma = which == stream_case::comma_delimited_small ||
which == stream_case::comma_delimited_large;
const size_t payload_size = small ? SMALL_PAYLOAD : LARGE_PAYLOAD;
const size_t count = TARGET_BYTES / (payload_size + 48);
std::string out;
out.reserve(count * (payload_size + 64));
for (size_t i = 0; i < count; i++) {
if (rfc) {
out += char(0x1E);
}
if (comma && i > 0) {
out += ',';
}
out += make_document(i, payload_size);
if (!comma) {
out += '\n';
}
}
return {padded_string(out), count};
}
const stream_dataset &get_dataset(stream_case which) {
static const stream_dataset ndjson_small =
build_dataset(stream_case::ndjson_small);
static const stream_dataset ndjson_large =
build_dataset(stream_case::ndjson_large);
static const stream_dataset rfc_small =
build_dataset(stream_case::rfc7464_small);
static const stream_dataset rfc_large =
build_dataset(stream_case::rfc7464_large);
static const stream_dataset comma_small =
build_dataset(stream_case::comma_delimited_small);
static const stream_dataset comma_large =
build_dataset(stream_case::comma_delimited_large);
switch (which) {
case stream_case::ndjson_small:
return ndjson_small;
case stream_case::ndjson_large:
return ndjson_large;
case stream_case::rfc7464_small:
return rfc_small;
case stream_case::rfc7464_large:
return rfc_large;
case stream_case::comma_delimited_small:
return comma_small;
case stream_case::comma_delimited_large:
return comma_large;
}
return ndjson_small;
}
void set_counters(benchmark::State &state, const stream_dataset &dataset) {
state.SetBytesProcessed(int64_t(state.iterations()) * int64_t(dataset.json.size()));
state.SetItemsProcessed(int64_t(state.iterations()) * int64_t(dataset.count));
}
template <stream_case which, bool threaded = true>
static void bench_ondemand(benchmark::State &state) {
const auto &dataset = get_dataset(which);
ondemand::parser parser;
parser.threaded = threaded;
stream_format format = stream_format::whitespace_delimited;
if constexpr (which == stream_case::rfc7464_small ||
which == stream_case::rfc7464_large) {
format = stream_format::json_sequence;
} else if constexpr (which == stream_case::comma_delimited_small ||
which == stream_case::comma_delimited_large) {
format = stream_format::comma_delimited;
}
for (const auto _ : state) {
ondemand::document_stream docs;
auto error = parser.iterate_many(dataset.json, BATCH_SIZE, format).get(docs);
if (error) {
state.SkipWithError(error_message(error));
return;
}
uint64_t sum = 0;
for (auto doc : docs) {
ondemand::object obj;
if ((error = doc.get_object().get(obj))) {
state.SkipWithError(error_message(error));
return;
}
uint64_t id;
if ((error = obj["id"].get_uint64().get(id))) {
state.SkipWithError(error_message(error));
return;
}
sum += id;
}
benchmark::DoNotOptimize(sum);
}
set_counters(state, dataset);
}
template <stream_case which>
static void bench_dom(benchmark::State &state) {
const auto &dataset = get_dataset(which);
dom::parser parser;
parser.threaded = true;
stream_format format = stream_format::whitespace_delimited;
if constexpr (which == stream_case::rfc7464_small ||
which == stream_case::rfc7464_large) {
format = stream_format::json_sequence;
} else if constexpr (which == stream_case::comma_delimited_small ||
which == stream_case::comma_delimited_large) {
format = stream_format::comma_delimited;
}
for (const auto _ : state) {
dom::document_stream docs;
auto error = parser.parse_many(dataset.json, BATCH_SIZE, format).get(docs);
if (error) {
state.SkipWithError(error_message(error));
return;
}
uint64_t sum = 0;
for (auto doc : docs) {
uint64_t id;
if ((error = doc["id"].get(id))) {
state.SkipWithError(error_message(error));
return;
}
sum += id;
}
benchmark::DoNotOptimize(sum);
}
set_counters(state, dataset);
}
} // namespace
BENCHMARK(bench_ondemand<stream_case::ndjson_small>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_ondemand<stream_case::ndjson_large>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_ondemand<stream_case::rfc7464_small>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_ondemand<stream_case::rfc7464_large>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_ondemand<stream_case::comma_delimited_small>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_ondemand<stream_case::comma_delimited_large>)
->UseRealTime()
->DisplayAggregatesOnly(true);
// Non-threaded comma_delimited for comparison
BENCHMARK(bench_ondemand<stream_case::comma_delimited_small, false>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_ondemand<stream_case::comma_delimited_large, false>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_dom<stream_case::ndjson_small>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_dom<stream_case::ndjson_large>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_dom<stream_case::rfc7464_small>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_dom<stream_case::rfc7464_large>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_dom<stream_case::comma_delimited_small>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK(bench_dom<stream_case::comma_delimited_large>)
->UseRealTime()
->DisplayAggregatesOnly(true);
BENCHMARK_MAIN();
doc/iterate_many.md
+87 -29
View File
@@ -132,7 +132,7 @@ E.g., `[1,2]{"32":1}` is recognized as two documents.
Some official formats **(non-exhaustive list)**:
- [Newline-Delimited JSON (NDJSON)](https://github.com/ndjson/ndjson-spec/)
- [JSON lines (JSONL)](http://jsonlines.org/)
- [Record separator-delimited JSON (RFC 7464)](https://tools.ietf.org/html/rfc7464) <- Not supported by simdjson!
- [Record separator-delimited JSON (RFC 7464)](https://tools.ietf.org/html/rfc7464)
- [More on Wikipedia...](https://en.wikipedia.org/wiki/JSON_streaming)
API
@@ -278,39 +278,97 @@ Importantly, you should only call `truncated_bytes()` after iterating through al
Comma-separated documents
-----------
We also support comma-separated documents, but with some performance limitations. The `iterate_many` function takes in an option to allow parsing of comma separated documents (which defaults on false). In this mode, the entire buffer is processed in one batch. Therefore, the total size of the document should not exceed the maximal capacity of the parser (4 GB). This mode also effectively disallow multithreading. It is therefore mostly suitable for not "very large" inputs. In this mode, the batch_size parameter
is effectively ignored, as it is set to at least the document size.
Example:
To parse comma-separated documents like `{"a":1},{"b":2},{"c":3}`, use the `stream_format::comma_delimited` parameter:
```cpp
auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
// We pass '32' as the batch size, but it is a bogus parameter because, since
// we pass 'true' to the allow_comma parameter, the batch size will be set to at least
// the document size.
auto error = parser.iterate_many(json, 32, true).get(doc_stream);
if (error) { std::cerr << error << std::endl; return; }
for (auto doc : doc_stream) {
std::cout << doc.type() << std::endl;
}
```
This will print:
auto json = R"({"a":1},{"b":2},{"c":3})"_padded;
ondemand::parser parser;
ondemand::document_stream stream;
auto error = parser.iterate_many(json, ondemand::DEFAULT_BATCH_SIZE,
simdjson::stream_format::comma_delimited).get(stream);
if (error) { std::cerr << error << std::endl; return; }
for (auto doc : stream) {
std::cout << doc << std::endl;
}
// Prints: {"a":1}
// {"b":2}
// {"c":3}
```
number
number
number
number
string
string
string
object
array
Whitespace around the commas is allowed:
```cpp
auto json = R"({"a":1} , {"b":2} , {"c":3})"_padded; // Also works
```
Nested commas inside objects and arrays are preserved:
```cpp
auto json = R"({"arr":[1,2,3]},{"obj":{"x":1,"y":2}})"_padded;
// Correctly parses as 2 documents, not 6
```
Mixed document types are supported:
```cpp
auto json = R"(1, 2, 3, 4, "a", "b", "c", {"hello": "world"}, [1, 2, 3])"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
auto error = parser.iterate_many(json, ondemand::DEFAULT_BATCH_SIZE,
simdjson::stream_format::comma_delimited).get(doc_stream);
if (error) { std::cerr << error << std::endl; return; }
for (auto doc : doc_stream) {
std::cout << doc.type() << std::endl;
}
// Prints: number number number number string string string object array
```
Extra top-level separators are tolerated for compatibility with the legacy
`allow_comma_separated` behavior. For example, leading commas, trailing commas,
and repeated commas are treated as empty separators rather than documents.
### Legacy `allow_comma_separated` parameter (deprecated)
The `allow_comma_separated` boolean parameter is deprecated. When set to `true`, it now internally maps to `stream_format::comma_delimited`.
The old single-batch limitation no longer applies - comma-delimited parsing now supports multi-batch processing and threading for optimal performance on large files.
JSON Text Sequences (RFC 7464)
------------------------------
[RFC 7464](https://tools.ietf.org/html/rfc7464) defines a format for streaming JSON values using ASCII Record Separator (RS, 0x1E) as a delimiter. Each JSON text is preceded by RS and optionally followed by ASCII Line Feed (LF, 0x0A).
Example input:
```
<RS>{"name":"doc1"}<LF>
<RS>{"name":"doc2"}<LF>
<RS>{"name":"doc3"}<LF>
```
To parse JSON text sequences, use the `stream_format::json_sequence` parameter:
```cpp
// Build input with RS (0x1E) and LF (0x0A) delimiters
std::string input_str;
input_str += '\x1e'; input_str += "{\"a\":1}"; input_str += '\x0a';
input_str += '\x1e'; input_str += "{\"b\":2}"; input_str += '\x0a';
input_str += '\x1e'; input_str += "{\"c\":3}"; input_str += '\x0a';
simdjson::padded_string input(input_str);
ondemand::parser parser;
ondemand::document_stream stream;
auto error = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE,
simdjson::stream_format::json_sequence).get(stream);
if (error) { std::cerr << error << std::endl; return; }
for (auto doc : stream) {
std::cout << doc << std::endl;
}
```
The `stream_format` enum has the following values:
- `stream_format::whitespace_delimited` (default): Standard NDJSON/JSON Lines format
- `stream_format::json_sequence`: RFC 7464 format with RS delimiters
- `stream_format::comma_delimited`: Comma-separated JSON documents
The trailing LF after each JSON text is optional but recommended by the RFC for robustness.
C++20 features
--------------------
doc/parse_many.md
+79 -1
View File
@@ -132,7 +132,7 @@ Whitespace Characters:
Some official formats **(non-exhaustive list)**:
- [Newline-Delimited JSON (NDJSON)](https://github.com/ndjson/ndjson-spec)
- [JSON lines (JSONL)](http://jsonlines.org/)
- [Record separator-delimited JSON (RFC 7464)](https://tools.ietf.org/html/rfc7464) <- Not supported by simdjson!
- [Record separator-delimited JSON (RFC 7464)](https://tools.ietf.org/html/rfc7464)
- [More on Wikipedia...](https://en.wikipedia.org/wiki/JSON_streaming)
API
@@ -253,3 +253,81 @@ Consider the following example where a truncated document (`{"key":"intentionall
Importantly, you should only call `truncated_bytes()` after iterating through all of the documents since the stream cannot tell whether there are truncated documents at the very end when it may not have accessed that part of the data yet.
JSON Text Sequences (RFC 7464)
------------------------------
[RFC 7464](https://tools.ietf.org/html/rfc7464) defines a format for streaming JSON values using ASCII Record Separator (RS, 0x1E) as a delimiter. Each JSON text is preceded by RS and optionally followed by ASCII Line Feed (LF, 0x0A).
Example input:
```
<RS>{"name":"doc1"}<LF>
<RS>{"name":"doc2"}<LF>
<RS>{"name":"doc3"}<LF>
```
To parse JSON text sequences, use the `stream_format::json_sequence` parameter:
```cpp
// Build input with RS (0x1E) and LF (0x0A) delimiters
std::string input_str;
input_str += '\x1e'; input_str += "{\"a\":1}"; input_str += '\x0a';
input_str += '\x1e'; input_str += "{\"b\":2}"; input_str += '\x0a';
input_str += '\x1e'; input_str += "{\"c\":3}"; input_str += '\x0a';
simdjson::padded_string input(input_str);
simdjson::dom::parser parser;
simdjson::dom::document_stream stream;
auto error = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE,
simdjson::stream_format::json_sequence).get(stream);
if (error) { std::cerr << error << std::endl; return; }
for (auto doc : stream) {
std::cout << doc << std::endl;
}
```
The `stream_format` enum has the following values:
- `stream_format::whitespace_delimited` (default): Standard NDJSON/JSON Lines format
- `stream_format::json_sequence`: RFC 7464 format with RS delimiters
- `stream_format::comma_delimited`: Comma-separated JSON documents
The trailing LF after each JSON text is optional but recommended by the RFC for robustness.
Comma-Separated Documents
-------------------------
Some systems produce JSON documents separated by commas, like `{"a":1},{"b":2},{"c":3}`. This is common when extracting elements from a JSON array or when APIs return comma-separated results.
To parse comma-separated documents, use the `stream_format::comma_delimited` parameter:
```cpp
auto json = R"({"a":1},{"b":2},{"c":3})"_padded;
simdjson::dom::parser parser;
simdjson::dom::document_stream stream;
auto error = parser.parse_many(json, simdjson::dom::DEFAULT_BATCH_SIZE,
simdjson::stream_format::comma_delimited).get(stream);
if (error) { std::cerr << error << std::endl; return; }
for (auto doc : stream) {
std::cout << doc << std::endl;
}
// Prints: {"a":1}
// {"b":2}
// {"c":3}
```
Whitespace around the commas is allowed:
```cpp
auto json = R"({"a":1} , {"b":2} , {"c":3})"_padded; // Also works
```
Nested commas inside objects and arrays are preserved:
```cpp
auto json = R"({"arr":[1,2,3]},{"obj":{"x":1,"y":2}})"_padded;
// Correctly parses as 2 documents, not 6
```
Extra top-level separators are tolerated for compatibility with the legacy
On-Demand comma-separated mode. Leading commas, trailing commas, and repeated
commas are treated as empty separators rather than documents.
Unlike the legacy `allow_comma_separated` parameter, `stream_format::comma_delimited` supports multi-batch processing and threading for optimal performance on large files.
include/simdjson/base.h
+9
View File
@@ -50,6 +50,15 @@ struct padded_string;
class padded_string_view;
enum class stage1_mode;
/**
* Stream format for parse_many/iterate_many.
*/
enum class stream_format {
whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL)
json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited)
comma_delimited ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`)
};
namespace internal {
template<typename T>
include/simdjson/dom/document_stream-inl.h
+31 -3
View File
@@ -89,12 +89,14 @@ simdjson_inline document_stream::document_stream(
dom::parser &_parser,
const uint8_t *_buf,
size_t _len,
size_t _batch_size
size_t _batch_size,
stream_format _format
) noexcept
: parser{&_parser},
buf{_buf},
len{_len},
batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size},
format{_format},
error{SUCCESS}
#ifdef SIMDJSON_THREADS_ENABLED
, use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
@@ -112,6 +114,7 @@ simdjson_inline document_stream::document_stream() noexcept
buf{nullptr},
len{0},
batch_size{0},
format{stream_format::whitespace_delimited},
error{UNINITIALIZED}
#ifdef SIMDJSON_THREADS_ENABLED
, use_thread(false)
@@ -274,10 +277,35 @@ inline size_t document_stream::next_batch_start() const noexcept {
inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept {
size_t remaining = len - _batch_start;
stage1_mode mode;
if (remaining <= batch_size) {
return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final);
// Final batch
switch (format) {
case stream_format::json_sequence:
mode = stage1_mode::json_sequence_final;
break;
case stream_format::comma_delimited:
mode = stage1_mode::comma_delimited_final;
break;
default:
mode = stage1_mode::streaming_final;
break;
}
return p.implementation->stage1(&buf[_batch_start], remaining, mode);
} else {
return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial);
// Partial batch
switch (format) {
case stream_format::json_sequence:
mode = stage1_mode::json_sequence_partial;
break;
case stream_format::comma_delimited:
mode = stage1_mode::comma_delimited_partial;
break;
default:
mode = stage1_mode::streaming_partial;
break;
}
return p.implementation->stage1(&buf[_batch_start], batch_size, mode);
}
}
include/simdjson/dom/document_stream.h
+5 -1
View File
@@ -206,12 +206,14 @@ private:
* @param buf is the raw byte buffer we need to process
* @param len is the length of the raw byte buffer in bytes
* @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document)
* @param format is the stream format
*/
simdjson_inline document_stream(
dom::parser &parser,
const uint8_t *buf,
size_t len,
size_t batch_size
size_t batch_size,
stream_format format = stream_format::whitespace_delimited
) noexcept;
/**
@@ -261,6 +263,8 @@ private:
const uint8_t *buf;
size_t len;
size_t batch_size;
/** The stream format. */
stream_format format;
/** The error (or lack thereof) from the current document. */
error_code error;
size_t batch_start{0};
include/simdjson/dom/parser-inl.h
+19 -6
View File
@@ -170,12 +170,7 @@ simdjson_inline simdjson_result<element> parser::parse(const padded_string_view
}
inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
buf += 3;
len -= 3;
}
return document_stream(*this, buf, len, batch_size);
return parse_many(buf, len, batch_size, stream_format::whitespace_delimited);
}
inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept {
return parse_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size);
@@ -187,6 +182,24 @@ inline simdjson_result<document_stream> parser::parse_many(const padded_string &
return parse_many(s.data(), s.length(), batch_size);
}
inline simdjson_result<document_stream> parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept {
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
buf += 3;
len -= 3;
}
return document_stream(*this, buf, len, batch_size, format);
}
inline simdjson_result<document_stream> parser::parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept {
return parse_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size, format);
}
inline simdjson_result<document_stream> parser::parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept {
return parse_many(s.data(), s.length(), batch_size, format);
}
inline simdjson_result<document_stream> parser::parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept {
return parse_many(s.data(), s.length(), batch_size, format);
}
simdjson_inline size_t parser::capacity() const noexcept {
return implementation ? implementation->capacity() : 0;
}
include/simdjson/dom/parser.h
+17
View File
@@ -494,6 +494,23 @@ public:
/** @private We do not want to allow implicit conversion from C string to std::string. */
simdjson_result<document_stream> parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete;
/**
* Parse a stream of JSON documents with explicit format specification.
*
* @param buf The concatenated JSON documents.
* @param len The length of the buffer.
* @param batch_size The batch size to use.
* @param format The stream format.
* @return A stream of documents, or an error.
*/
inline simdjson_result<document_stream> parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept;
/**
* Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length
* and `max_depth` depth.
include/simdjson/generic/ondemand/document_stream-inl.h
+37 -6
View File
@@ -95,13 +95,15 @@ simdjson_inline document_stream::document_stream(
const uint8_t *_buf,
size_t _len,
size_t _batch_size,
bool _allow_comma_separated
bool _allow_comma_separated,
stream_format _format
) noexcept
: parser{&_parser},
buf{_buf},
len{_len},
batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size},
allow_comma_separated{_allow_comma_separated},
format{_format},
error{SUCCESS}
#ifdef SIMDJSON_THREADS_ENABLED
, use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change
@@ -120,6 +122,7 @@ simdjson_inline document_stream::document_stream() noexcept
len{0},
batch_size{0},
allow_comma_separated{false},
format{stream_format::whitespace_delimited},
error{UNINITIALIZED}
#ifdef SIMDJSON_THREADS_ENABLED
, use_thread(false)
@@ -219,7 +222,10 @@ inline void document_stream::start() noexcept {
error = run_stage1(*parser, batch_start);
}
if (error) { return; }
doc_index = batch_start;
// For json_sequence mode, structural_indexes[0] points to the actual JSON value
// after the RS delimiter and any following whitespace. For regular mode, it is
// the offset from batch_start to the first document in the batch.
doc_index = batch_start + parser->implementation->structural_indexes[0];
doc = document(json_iterator(&buf[batch_start], parser));
doc.iter._streaming = true;
@@ -300,7 +306,7 @@ inline void document_stream::next() noexcept {
*/
if (error) { continue; } // If the error was EMPTY, we may want to load another batch.
doc_index = batch_start;
doc_index = batch_start + parser->implementation->structural_indexes[0];
}
}
}
@@ -329,10 +335,35 @@ inline error_code document_stream::run_stage1(ondemand::parser &p, size_t _batch
// This code only updates the structural index in the parser, it does not update any json_iterator
// instance.
size_t remaining = len - _batch_start;
stage1_mode mode;
if (remaining <= batch_size) {
return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final);
// Final batch
switch (format) {
case stream_format::json_sequence:
mode = stage1_mode::json_sequence_final;
break;
case stream_format::comma_delimited:
mode = stage1_mode::comma_delimited_final;
break;
default:
mode = stage1_mode::streaming_final;
break;
}
return p.implementation->stage1(&buf[_batch_start], remaining, mode);
} else {
return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial);
// Partial batch
switch (format) {
case stream_format::json_sequence:
mode = stage1_mode::json_sequence_partial;
break;
case stream_format::comma_delimited:
mode = stage1_mode::comma_delimited_partial;
break;
default:
mode = stage1_mode::streaming_partial;
break;
}
return p.implementation->stage1(&buf[_batch_start], batch_size, mode);
}
}
@@ -441,4 +472,4 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::document_stre
}
#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H
#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H
include/simdjson/generic/ondemand/document_stream.h
+5 -1
View File
@@ -229,13 +229,16 @@ private:
* @param buf is the raw byte buffer we need to process
* @param len is the length of the raw byte buffer in bytes
* @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document)
* @param allow_comma_separated whether to allow comma-separated documents
* @param format the stream format
*/
simdjson_inline document_stream(
ondemand::parser &parser,
const uint8_t *buf,
size_t len,
size_t batch_size,
bool allow_comma_separated
bool allow_comma_separated,
stream_format format = stream_format::whitespace_delimited
) noexcept;
/**
@@ -284,6 +287,7 @@ private:
size_t len;
size_t batch_size;
bool allow_comma_separated;
stream_format format;
/**
* We are going to use just one document instance. The document owns
* the json_iterator. It implies that we only ever pass a reference
include/simdjson/generic/ondemand/parser-inl.h
+57 -2
View File
@@ -133,6 +133,34 @@ simdjson_warn_unused simdjson_inline simdjson_result<json_iterator> parser::iter
return json_iterator(reinterpret_cast<const uint8_t *>(json.data()), this);
}
inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept {
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
buf += 3;
len -= 3;
}
return document_stream(*this, buf, len, batch_size, false, stream_format::whitespace_delimited);
}
inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size) noexcept {
return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size);
}
inline simdjson_result<document_stream> parser::iterate_many(padded_string_view s, size_t batch_size) noexcept {
if (!s.has_sufficient_padding()) { return INSUFFICIENT_PADDING; }
return iterate_many(s.data(), s.length(), batch_size);
}
inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size) noexcept {
return iterate_many(padded_string_view(s), batch_size);
}
inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size) noexcept {
return iterate_many(padded_string_view(s), batch_size);
}
inline simdjson_result<document_stream> parser::iterate_many(std::string &s, size_t batch_size) noexcept {
return iterate_many(pad(s), batch_size);
}
#ifndef SIMDJSON_DISABLE_DEPRECATED_API
SIMDJSON_PUSH_DISABLE_WARNINGS
SIMDJSON_DISABLE_DEPRECATED_WARNING
inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
// Warning: no check is done on the buffer padding. We trust the user.
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
@@ -140,8 +168,11 @@ inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf,
buf += 3;
len -= 3;
}
if(allow_comma_separated && batch_size < len) { batch_size = len; }
return document_stream(*this, buf, len, batch_size, allow_comma_separated);
// Map allow_comma_separated to stream_format::comma_delimited
if (allow_comma_separated) {
return document_stream(*this, buf, len, batch_size, false, stream_format::comma_delimited);
}
return document_stream(*this, buf, len, batch_size, false, stream_format::whitespace_delimited);
}
inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept {
@@ -161,6 +192,30 @@ inline simdjson_result<document_stream> parser::iterate_many(const std::string &
inline simdjson_result<document_stream> parser::iterate_many(std::string &s, size_t batch_size, bool allow_comma_separated) noexcept {
return iterate_many(pad(s), batch_size, allow_comma_separated);
}
SIMDJSON_POP_DISABLE_WARNINGS
#endif // SIMDJSON_DISABLE_DEPRECATED_API
inline simdjson_result<document_stream> parser::iterate_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept {
if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; }
if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) {
buf += 3;
len -= 3;
}
return document_stream(*this, buf, len, batch_size, false, format);
}
inline simdjson_result<document_stream> parser::iterate_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept {
return iterate_many(reinterpret_cast<const uint8_t *>(buf), len, batch_size, format);
}
inline simdjson_result<document_stream> parser::iterate_many(padded_string_view s, size_t batch_size, stream_format format) noexcept {
if (!s.has_sufficient_padding()) { return INSUFFICIENT_PADDING; }
return iterate_many(s.data(), s.length(), batch_size, format);
}
inline simdjson_result<document_stream> parser::iterate_many(const std::string &s, size_t batch_size, stream_format format) noexcept {
return iterate_many(padded_string_view(s), batch_size, format);
}
inline simdjson_result<document_stream> parser::iterate_many(const padded_string &s, size_t batch_size, stream_format format) noexcept {
return iterate_many(padded_string_view(s), batch_size, format);
}
simdjson_pure simdjson_inline size_t parser::capacity() const noexcept {
return _capacity;
}
include/simdjson/generic/ondemand/parser.h
+51 -17
View File
@@ -244,32 +244,66 @@ public:
* spot is cache-related: small enough to fit in cache, yet big enough to
* parse as many documents as possible in one tight loop.
* Defaults to 10MB, which has been a reasonable sweet spot in our tests.
* @param allow_comma_separated (defaults on false) This allows a mode where the documents are
* separated by commas instead of whitespace. It comes with a performance
* penalty because the entire document is indexed at once (and the document must be
* less than 4 GB), and there is no multithreading. In this mode, the batch_size parameter
* is effectively ignored, as it is set to at least the document size.
* @param allow_comma_separated @deprecated Use stream_format::comma_delimited instead.
* When true, maps internally to stream_format::comma_delimited.
* Defaults to false.
* @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors:
* - MEMALLOC if the parser does not have enough capacity and memory allocation fails
* - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity.
* - other json errors if parsing fails. You should not rely on these errors to always the same for the
* same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware).
*/
inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size)
inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(padded_string_view json, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size)
the string might be automatically padded with up to SIMDJSON_PADDING whitespace characters */
inline simdjson_result<document_stream> iterate_many(std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
/** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept;
inline simdjson_result<document_stream> iterate_many(std::string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size) */
inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept;
/** @private We do not want to allow implicit conversion from C string to std::string. */
simdjson_result<document_stream> iterate_many(const char *buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete;
#ifndef SIMDJSON_DISABLE_DEPRECATED_API
/**
* @deprecated Use iterate_many with stream_format::comma_delimited instead.
*/
simdjson_deprecated inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) */
simdjson_deprecated inline simdjson_result<document_stream> iterate_many(padded_string_view json, size_t batch_size, bool allow_comma_separated) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) */
simdjson_deprecated inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) */
simdjson_deprecated inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size, bool allow_comma_separated) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) */
simdjson_deprecated inline simdjson_result<document_stream> iterate_many(std::string &s, size_t batch_size, bool allow_comma_separated) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, bool allow_comma_separated) */
simdjson_deprecated inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size, bool allow_comma_separated) noexcept;
#endif // SIMDJSON_DISABLE_DEPRECATED_API
/**
* Parse a stream of JSON documents with explicit format specification.
*
* @param buf The concatenated JSON documents.
* @param len The length of the buffer.
* @param batch_size The batch size to use.
* @param format The stream format (whitespace_delimited, json_sequence, or comma_delimited).
* @return A stream of documents, or an error.
*/
inline simdjson_result<document_stream> iterate_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> iterate_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> iterate_many(padded_string_view s, size_t batch_size, stream_format format) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> iterate_many(const std::string &s, size_t batch_size, stream_format format) noexcept;
/** @overload iterate_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */
inline simdjson_result<document_stream> iterate_many(const padded_string &s, size_t batch_size, stream_format format) noexcept;
/** The capacity of this parser (the largest document it can process). */
simdjson_pure simdjson_inline size_t capacity() const noexcept;
/** The maximum capacity of this parser (the largest document it is allowed to process). */
@@ -425,4 +459,4 @@ public:
} // namespace simdjson
#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_H
#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_H
include/simdjson/internal/dom_parser_implementation.h
+6 -2
View File
@@ -19,7 +19,12 @@ class document;
* 3) The stream_final mode allows us to truncate final
* unterminated strings. It is useful in conjunction with streaming_partial.
*/
enum class stage1_mode { regular, streaming_partial, streaming_final};
enum class stage1_mode {
regular,
streaming_partial, streaming_final,
json_sequence_partial, json_sequence_final,
comma_delimited_partial, comma_delimited_final
};
/**
* Returns true if mode == streaming_partial or mode == streaming_final
@@ -31,7 +36,6 @@ inline bool is_streaming(stage1_mode mode) {
// return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final);
}
namespace internal {
src/fallback.cpp
+50
View File
@@ -283,6 +283,56 @@ simdjson_warn_unused simdjson_inline error_code scan() {
// doing.
parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len);
if (parser.n_structural_indexes == 0) { return EMPTY; }
} else if (partial == stage1_mode::json_sequence_partial) {
// RFC 7464: use RS positions for batch boundaries
if(unclosed_string) {
parser.n_structural_indexes--;
if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return CAPACITY; }
}
uint32_t next_batch_start = uint32_t(len);
auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start);
if (new_structural_indexes == DOCUMENT_TOO_LARGE) {
return CAPACITY;
}
if (new_structural_indexes == 0) {
parser.n_structural_indexes = 0;
return EMPTY;
}
parser.n_structural_indexes = new_structural_indexes;
parser.structural_indexes[parser.n_structural_indexes] = next_batch_start;
} else if (partial == stage1_mode::json_sequence_final) {
// RFC 7464: final batch, last document extends to EOF
if(unclosed_string) { parser.n_structural_indexes--; }
uint32_t next_batch_start = uint32_t(len);
parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start);
parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes];
parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len);
if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return EMPTY; }
} else if (partial == stage1_mode::comma_delimited_partial) {
// Comma-delimited: filter root-level commas, use comma positions for batch boundaries
if(unclosed_string) {
parser.n_structural_indexes--;
if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return CAPACITY; }
}
uint32_t next_batch_start = uint32_t(len);
auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start);
if (new_structural_indexes == DOCUMENT_TOO_LARGE) {
return CAPACITY;
}
if (new_structural_indexes == 0) {
parser.n_structural_indexes = 0;
return EMPTY;
}
parser.n_structural_indexes = new_structural_indexes;
parser.structural_indexes[parser.n_structural_indexes] = next_batch_start;
} else if (partial == stage1_mode::comma_delimited_final) {
// Comma-delimited: final batch, last document extends to EOF
if(unclosed_string) { parser.n_structural_indexes--; }
uint32_t next_batch_start = uint32_t(len);
parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start);
parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes];
parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len);
if (simdjson_unlikely(parser.n_structural_indexes == 0)) { return EMPTY; }
} else if(unclosed_string) { error = UNCLOSED_STRING; }
return error;
}
src/generic/stage1/find_next_document_index.h
+234 -1
View File
@@ -97,9 +97,242 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par
return 0;
}
/**
* Sentinel value returned to indicate a document started but didn't fit
* (CAPACITY error), as opposed to 0 which means no document content found
* (EMPTY).
*/
constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX;
/**
* For RFC 7464 JSON text sequences, filter RS from structural indexes and
* find batch boundaries.
*
* In JSON sequence mode, RS (0x1E) marks the start of each JSON text.
* RS bytes appear in structural_indexes as they are classified as scalars.
* This function:
* 1. Scans structural_indexes to find and count RS positions
* 2. Filters RS out of structural_indexes in-place
* 3. Determines batch boundaries based on RS positions
*
* @param parser The parser with structural_indexes and buf.
* @param len The length of the current batch buffer.
* @param is_final True if this is the final batch (no more data coming).
* @param next_batch_start Output: offset where the next batch should start.
* @return The number of structural indexes to keep (after RS filtering),
* 0 if no document content found (EMPTY),
* or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY).
*/
simdjson_inline uint32_t find_next_document_index_json_sequence(
dom_parser_implementation &parser,
size_t len,
bool is_final,
uint32_t &next_batch_start) {
// Default: next batch starts at end of buffer
next_batch_start = uint32_t(len);
if (parser.n_structural_indexes == 0) { return 0; }
// Phase 1: Scan structural_indexes to find RS positions and handle them.
// RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS
// is already in structural_indexes (it's an operator). For scalars like numbers,
// the digit following RS is NOT in structural_indexes because the scanner sees
// RS as a scalar, making the digit a scalar continuation, not a start.
// We must: (1) remove RS from structural_indexes, and (2) for scalars, add the
// actual value start position.
uint32_t write_idx = 0;
uint32_t last_rs_pos = 0;
uint32_t rs_count = 0;
for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) {
const uint32_t pos = parser.structural_indexes[read_idx];
if (parser.buf[pos] == 0x1E) {
// This is an RS character - find the actual JSON value start
last_rs_pos = pos;
rs_count++;
// Skip RS and any whitespace to find the actual value start
uint32_t value_start = pos + 1;
while (value_start < len) {
const uint8_t c = parser.buf[value_start];
if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { value_start++; }
else { break; }
}
// Check if the value start is an operator (already in structural_indexes),
// another RS (empty record), or a scalar (needs to be added).
if (value_start < len) {
const uint8_t c = parser.buf[value_start];
const bool is_operator = (c == '{' || c == '}' || c == '[' || c == ']' || c == ':' || c == ',');
const bool is_rs = (c == 0x1E);
if (!is_operator && !is_rs) {
// Scalar value - add its position since scanner missed it
parser.structural_indexes[write_idx++] = value_start;
}
// For operators, they will appear later in structural_indexes
// For RS, it will be processed in a subsequent iteration
}
} else {
// Not RS, copy to output
parser.structural_indexes[write_idx++] = pos;
}
}
// Update structural index count
parser.n_structural_indexes = write_idx;
if (parser.n_structural_indexes == 0) { return 0; }
if (rs_count == 0) {
// No RS found; for final batch, try generic boundary detection
return is_final ? find_next_document_index(parser) : 0;
}
// Phase 2: Determine batch boundaries based on RS positions
if (is_final) {
// Final batch: all documents are complete (last one ends at EOF).
// In json_sequence mode, RS markers define document boundaries, so all
// remaining structurals form complete documents. Return them all directly.
// (Calling find_next_document_index() would fail for scalar documents.)
return parser.n_structural_indexes;
}
// Partial batch: need to find complete documents only.
// A document starting at an RS is complete if there is another RS after it.
next_batch_start = last_rs_pos;
if (rs_count < 2) {
// Only one RS, so we have at most one document that may be incomplete.
// We cannot confirm it is complete without another RS.
// Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators.
return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0;
}
// We have at least 2 RS markers. The last complete document ends before last_rs_pos.
// Find the structural index cutoff: keep only structurals < last_rs_pos.
// Since we already filtered RS, all remaining structurals are valid.
// We iterate backward to find the last structural before last_rs_pos.
uint32_t keep_count = 0;
for (uint32_t i = parser.n_structural_indexes; i > 0; i--) {
if (parser.structural_indexes[i - 1] < last_rs_pos) {
keep_count = i;
break;
}
}
// No structurals before the last RS - no complete documents
if (keep_count == 0) { return 0; }
// All documents before the last RS are complete by definition (the next RS
// confirms their end). No need to call find_next_document_index() which
// would fail for scalar documents like `1` or `"hello"`.
return keep_count;
}
/**
* Filter comma-delimited documents by removing root-level commas from
* structural indexes.
*
* For comma-delimited format like `{...},{...},{...}`, we need to remove
* the commas that separate documents (depth 0) while preserving commas
* inside arrays and objects (depth > 0).
*
* After filtering, the structural indexes look like whitespace-delimited
* documents, so find_next_document_index() works unchanged.
*
* @param parser The parser with structural_indexes and buf.
* @param len The length of the current batch buffer.
* @param is_final True if this is the final batch (no more data coming).
* @param next_batch_start Output: offset where the next batch should start.
* @return The number of structural indexes to keep,
* 0 if no document content found (EMPTY),
* or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY).
*/
simdjson_inline uint32_t filter_comma_delimited(
dom_parser_implementation &parser,
size_t len,
bool is_final,
uint32_t &next_batch_start) {
// Default: next batch starts at end of buffer
next_batch_start = uint32_t(len);
if (parser.n_structural_indexes == 0) { return 0; }
// Track depth to identify root-level commas (depth 0)
int depth = 0;
uint32_t write_idx = 0;
uint32_t last_root_comma_pos = 0;
uint32_t root_comma_count = 0;
for (uint32_t i = 0; i < parser.n_structural_indexes; i++) {
uint32_t idx = parser.structural_indexes[i];
uint8_t c = parser.buf[idx];
switch (c) {
case '{': case '[':
depth++;
parser.structural_indexes[write_idx++] = idx;
break;
case '}': case ']':
depth--;
parser.structural_indexes[write_idx++] = idx;
break;
case ',':
if (depth == 0) {
// Root-level comma = document boundary, skip it
last_root_comma_pos = idx;
root_comma_count++;
continue;
}
parser.structural_indexes[write_idx++] = idx;
break;
default:
// Colons, scalars, etc.
parser.structural_indexes[write_idx++] = idx;
break;
}
}
// Update structural index count
parser.n_structural_indexes = write_idx;
if (parser.n_structural_indexes == 0) { return 0; }
if (is_final) {
// Final batch: use standard boundary detection on filtered indexes
return find_next_document_index(parser);
}
// Partial batch: need to find complete documents only.
// A document ending with a root comma is complete.
if (root_comma_count == 0) {
// No root commas found; we cannot confirm any document is complete.
// The whole batch might be one incomplete document.
// Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas.
return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0;
}
// We have at least one root comma. Documents before the last comma are complete.
next_batch_start = last_root_comma_pos + 1;
// Find the structural index cutoff: keep only structurals < last_root_comma_pos
uint32_t keep_count = 0;
for (uint32_t i = parser.n_structural_indexes; i > 0; i--) {
if (parser.structural_indexes[i - 1] < last_root_comma_pos) {
keep_count = i;
break;
}
}
if (keep_count == 0) { return 0; }
// Use standard boundary detection on the complete portion
parser.n_structural_indexes = keep_count;
return find_next_document_index(parser);
}
} // namespace stage1
} // unnamed namespace
} // namespace SIMDJSON_IMPLEMENTATION
} // namespace simdjson
#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H
#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H
src/generic/stage1/json_structural_indexer.h
+50 -1
View File
@@ -314,7 +314,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa
return EMPTY;
}
}
parser.n_structural_indexes = new_structural_indexes;
} else if (partial == stage1_mode::streaming_final) {
if(have_unclosed_string) { parser.n_structural_indexes--; }
@@ -342,6 +341,56 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa
// the trailing garbage.
return EMPTY;
}
} else if (partial == stage1_mode::json_sequence_partial) {
// RFC 7464: use RS positions for batch boundaries
if(have_unclosed_string) {
parser.n_structural_indexes--;
if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; }
}
uint32_t next_batch_start = uint32_t(len);
auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start);
if (new_structural_indexes == DOCUMENT_TOO_LARGE) {
return CAPACITY;
}
if (new_structural_indexes == 0) {
parser.n_structural_indexes = 0;
return EMPTY;
}
parser.n_structural_indexes = new_structural_indexes;
parser.structural_indexes[parser.n_structural_indexes] = next_batch_start;
} else if (partial == stage1_mode::json_sequence_final) {
// RFC 7464: final batch, last document extends to EOF
if(have_unclosed_string) { parser.n_structural_indexes--; }
uint32_t next_batch_start = uint32_t(len);
parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start);
parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes];
parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len);
if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; }
} else if (partial == stage1_mode::comma_delimited_partial) {
// Comma-delimited: filter root-level commas, use comma positions for batch boundaries
if(have_unclosed_string) {
parser.n_structural_indexes--;
if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; }
}
uint32_t next_batch_start = uint32_t(len);
auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start);
if (new_structural_indexes == DOCUMENT_TOO_LARGE) {
return CAPACITY;
}
if (new_structural_indexes == 0) {
parser.n_structural_indexes = 0;
return EMPTY;
}
parser.n_structural_indexes = new_structural_indexes;
parser.structural_indexes[parser.n_structural_indexes] = next_batch_start;
} else if (partial == stage1_mode::comma_delimited_final) {
// Comma-delimited: final batch, last document extends to EOF
if(have_unclosed_string) { parser.n_structural_indexes--; }
uint32_t next_batch_start = uint32_t(len);
parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start);
parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes];
parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len);
if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; }
}
checker.check_eof();
return checker.errors();
tests/dom/document_stream_tests.cpp
+281 -2
View File
@@ -983,8 +983,287 @@ namespace document_stream_tests {
return true;
}
bool json_sequence_tests() {
TEST_START();
simdjson::dom::parser parser;
simdjson::dom::document_stream stream;
// Verify enum values
static_assert(static_cast<int>(simdjson::stream_format::whitespace_delimited) == 0, "whitespace_delimited should be 0");
static_assert(static_cast<int>(simdjson::stream_format::json_sequence) == 1, "json_sequence should be 1");
SUBTEST("whitespace_delimited format works", ([&]() {
auto input = R"({"a":1} {"b":2} {"c":3})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::whitespace_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("basic RS-delimited objects with LF", ([&]() {
auto input = simdjson::padded_string("\x1e{\"a\":1}\n\x1e{\"b\":2}\n\x1e{\"c\":3}\n"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
const char* keys[] = {"a", "b", "c"};
int64_t expected[] = {1, 2, 3};
size_t count = 0;
for (auto doc : stream) {
ASSERT_SUCCESS(doc.error());
int64_t value;
ASSERT_SUCCESS(doc[keys[count]].get(value));
ASSERT_EQUAL(value, expected[count]);
count++;
}
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("multiple documents without trailing LF", ([&]() {
auto input = simdjson::padded_string("\x1e{\"a\":1}\x1e{\"b\":2}\x1e{\"c\":3}"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("single document without LF", ([&]() {
auto input = simdjson::padded_string("\x1e{\"a\":1}"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(1));
return true;
}()));
SUBTEST("scalar documents", ([&]() {
auto input = simdjson::padded_string("\x1e" "42\n" "\x1e" "\"hello\"\n" "\x1e" "true\n" "\x1e" "null\n" "\x1e" "[1,2,3]\n"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(5));
return true;
}()));
SUBTEST("current_index points to JSON value not RS", ([&]() {
// Layout: RS(0) 1(1) LF(2) RS(3) 2(4) LF(5)
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "2\n"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(1)); // "1" at position 1
++it;
ASSERT_EQUAL(it.current_index(), size_t(4)); // "2" at position 4
return true;
}()));
SUBTEST("RS-only input produces 0 documents", ([&]() {
auto input = simdjson::padded_string("\x1e\n\x1e\n"s);
auto result = parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream);
if (result == simdjson::SUCCESS) {
size_t count = 0;
for (auto doc : stream) { (void)doc; count++; }
ASSERT_EQUAL(count, size_t(0));
}
// EMPTY error is also acceptable
return true;
}()));
SUBTEST("whitespace between RS and value", ([&]() {
auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t\n{\"b\":2}\n"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(2));
return true;
}()));
SUBTEST("mixed document types in sequence", ([&]() {
auto input = simdjson::padded_string("\x1e" "123\n" "\x1e" "{\"x\":1}\n" "\x1e" "[1,2]\n" "\x1e" "\"str\"\n"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
// Doc 0: number
int64_t num; ASSERT_SUCCESS((*it).get(num)); ASSERT_EQUAL(num, int64_t(123)); ++it;
// Doc 1: object
int64_t x; ASSERT_SUCCESS((*it)["x"].get(x)); ASSERT_EQUAL(x, int64_t(1)); ++it;
// Doc 2: array
simdjson::dom::array arr; ASSERT_SUCCESS((*it).get(arr)); ++it;
// Doc 3: string
std::string_view s; ASSERT_SUCCESS((*it).get(s)); ASSERT_EQUAL(s, std::string_view("str"));
return true;
}()));
SUBTEST("source returns correct document slice", ([&]() {
auto input = simdjson::padded_string("\x1e" "[1,2,3]\n" "\x1e" "{\"a\":1}\n"s);
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.source(), std::string_view("[1,2,3]"));
++it;
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
return true;
}()));
SUBTEST("multibatch current_index and source", ([&]() {
auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t{\"b\":2}\n"s);
ASSERT_SUCCESS(parser.parse_many(input, 10, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(2));
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
++it;
ASSERT_EQUAL(it.current_index(), size_t(12));
ASSERT_EQUAL(it.source(), std::string_view("{\"b\":2}"));
return true;
}()));
SUBTEST("small batch reports capacity", ([&]() {
std::string json_sequence_input = "\x1e[";
for (size_t i = 0; i < 2000; i++) {
json_sequence_input += '1';
json_sequence_input += (i + 1 < 2000 ? ',' : ']');
}
json_sequence_input += '\n';
auto input = simdjson::padded_string(json_sequence_input);
ASSERT_SUCCESS(parser.parse_many(input, 1024, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
auto doc = *it;
ASSERT_ERROR(doc.error(), simdjson::CAPACITY);
return true;
}()));
TEST_SUCCEED();
}
bool comma_delimited_tests() {
TEST_START();
simdjson::dom::parser parser;
simdjson::dom::document_stream stream;
SUBTEST("basic comma-separated objects", ([&]() {
auto input = R"({"a":1},{"b":2},{"c":3})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
const char* keys[] = {"a", "b", "c"};
int64_t expected[] = {1, 2, 3};
size_t count = 0;
for (auto doc : stream) {
ASSERT_SUCCESS(doc.error());
int64_t value;
ASSERT_SUCCESS(doc[keys[count]].get(value));
ASSERT_EQUAL(value, expected[count]);
count++;
}
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("comma-separated with whitespace", ([&]() {
auto input = R"({"a":1} , {"b":2} , {"c":3})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("comma-separated arrays", ([&]() {
auto input = R"([1,2],[3,4],[5,6])"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("mixed document types", ([&]() {
auto input = R"({"a":1},[2,3],"string",42,true,null)"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(6));
return true;
}()));
SUBTEST("nested commas preserved", ([&]() {
auto input = R"({"a":[1,2,3]},{"b":{"c":4,"d":5}})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
// First doc: {"a":[1,2,3]}
simdjson::dom::array arr;
ASSERT_SUCCESS((*it)["a"].get(arr));
size_t arr_count = 0;
for (auto elem : arr) { (void)elem; arr_count++; }
ASSERT_EQUAL(arr_count, size_t(3));
++it;
// Second doc: {"b":{"c":4,"d":5}}
int64_t c_val, d_val;
ASSERT_SUCCESS((*it)["b"]["c"].get(c_val));
ASSERT_SUCCESS((*it)["b"]["d"].get(d_val));
ASSERT_EQUAL(c_val, int64_t(4));
ASSERT_EQUAL(d_val, int64_t(5));
return true;
}()));
SUBTEST("single document no comma", ([&]() {
auto input = R"({"a":1})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(1));
return true;
}()));
SUBTEST("current_index points to JSON value", ([&]() {
// Layout: {(0) "(1) a(2) "(3) :(4) 1(5) }(6) ,(7) {(8) ...
auto input = R"({"a":1},{"b":2})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(0)); // First { at position 0
++it;
ASSERT_EQUAL(it.current_index(), size_t(8)); // Second { at position 8
return true;
}()));
SUBTEST("source returns correct document slice", ([&]() {
auto input = R"([1,2,3],{"a":1})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, simdjson::dom::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.source(), std::string_view("[1,2,3]"));
++it;
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
return true;
}()));
SUBTEST("multibatch current_index and source", ([&]() {
auto input = R"({"a":1}, {"b":2})"_padded;
ASSERT_SUCCESS(parser.parse_many(input, 10, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(0));
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
++it;
ASSERT_EQUAL(it.current_index(), size_t(9));
ASSERT_EQUAL(it.source(), std::string_view("{\"b\":2}"));
return true;
}()));
SUBTEST("small batch reports capacity", ([&]() {
std::string comma_input = " [";
for (size_t i = 0; i < 2000; i++) {
comma_input += '1';
comma_input += (i + 1 < 2000 ? ',' : ']');
}
auto input = simdjson::padded_string(comma_input);
ASSERT_SUCCESS(parser.parse_many(input, 1024, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
auto doc = *it;
ASSERT_ERROR(doc.error(), simdjson::CAPACITY);
return true;
}()));
TEST_SUCCEED();
}
bool run() {
return issue2181() &&
return json_sequence_tests() &&
comma_delimited_tests() &&
issue2181() &&
issue2170() &&
skipbom() &&
fuzzaccess() &&
@@ -1020,8 +1299,8 @@ namespace document_stream_tests {
document_stream_test() &&
document_stream_utf8_test();
}
}
} // namespace document_stream_tests
int main(int argc, char *argv[]) {
tests/ondemand/ondemand_document_stream_tests.cpp
+308
View File
@@ -945,8 +945,315 @@ namespace document_stream_tests {
TEST_SUCCEED();
}
bool json_sequence_tests() {
TEST_START();
ondemand::parser parser;
ondemand::document_stream stream;
// Verify enum values
static_assert(static_cast<int>(simdjson::stream_format::whitespace_delimited) == 0, "whitespace_delimited should be 0");
static_assert(static_cast<int>(simdjson::stream_format::json_sequence) == 1, "json_sequence should be 1");
SUBTEST("whitespace_delimited format works", ([&]() {
auto input = R"({"a":1} {"b":2} {"c":3})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::whitespace_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("basic RS-delimited objects with LF", ([&]() {
auto input = simdjson::padded_string("\x1e{\"a\":1}\n\x1e{\"b\":2}\n\x1e{\"c\":3}\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
const char* keys[] = {"a", "b", "c"};
int64_t expected[] = {1, 2, 3};
size_t count = 0;
for (auto doc : stream) {
ASSERT_SUCCESS(doc.error());
int64_t value;
ASSERT_SUCCESS(doc[keys[count]].get(value));
ASSERT_EQUAL(value, expected[count]);
count++;
}
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("multiple documents without trailing LF", ([&]() {
auto input = simdjson::padded_string("\x1e{\"a\":1}\x1e{\"b\":2}\x1e{\"c\":3}"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("single document without LF", ([&]() {
auto input = simdjson::padded_string("\x1e{\"a\":1}"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(1));
return true;
}()));
SUBTEST("scalar documents", ([&]() {
auto input = simdjson::padded_string("\x1e" "42\n" "\x1e" "\"hello\"\n" "\x1e" "true\n" "\x1e" "null\n" "\x1e" "[1,2,3]\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(5));
return true;
}()));
SUBTEST("current_index points to JSON value not RS", ([&]() {
// Layout: RS(0) 1(1) LF(2) RS(3) 2(4) LF(5)
auto input = simdjson::padded_string("\x1e" "1\n" "\x1e" "2\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(1)); // "1" at position 1
++it;
ASSERT_EQUAL(it.current_index(), size_t(4)); // "2" at position 4
return true;
}()));
SUBTEST("RS-only input produces 0 documents", ([&]() {
auto input = simdjson::padded_string("\x1e\n\x1e\n"s);
auto result = parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream);
// We accept either EMPTY error or successful iteration with 0 documents
if (result == simdjson::SUCCESS) {
size_t count = 0;
for (auto doc : stream) { (void)doc; count++; }
ASSERT_EQUAL(count, size_t(0));
}
// EMPTY error is also acceptable
return true;
}()));
SUBTEST("whitespace between RS and value", ([&]() {
auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t\n{\"b\":2}\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(2));
return true;
}()));
SUBTEST("mixed document types in sequence", ([&]() {
auto input = simdjson::padded_string("\x1e" "123\n" "\x1e" "{\"x\":1}\n" "\x1e" "[1,2]\n" "\x1e" "\"str\"\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
// Doc 0: number
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
int64_t num; ASSERT_SUCCESS(doc.get_int64().get(num)); ASSERT_EQUAL(num, int64_t(123));
}
++it;
// Doc 1: object
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
int64_t x; ASSERT_SUCCESS(doc["x"].get(x)); ASSERT_EQUAL(x, int64_t(1));
}
++it;
// Doc 2: array
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
ondemand::array arr; ASSERT_SUCCESS(doc.get_array().get(arr));
}
++it;
// Doc 3: string
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
std::string_view sv; ASSERT_SUCCESS(doc.get_string().get(sv)); ASSERT_EQUAL(sv, std::string_view("str"));
}
return true;
}()));
SUBTEST("source returns correct document slice", ([&]() {
auto input = simdjson::padded_string("\x1e" "[1,2,3]\n" "\x1e" "{\"a\":1}\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.source(), std::string_view("[1,2,3]"));
++it;
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
return true;
}()));
SUBTEST("multibatch current_index and source", ([&]() {
auto input = simdjson::padded_string("\x1e {\"a\":1}\n\x1e\t{\"b\":2}\n"s);
ASSERT_SUCCESS(parser.iterate_many(input, 10, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(2));
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
++it;
ASSERT_EQUAL(it.current_index(), size_t(12));
ASSERT_EQUAL(it.source(), std::string_view("{\"b\":2}"));
return true;
}()));
SUBTEST("small batch reports capacity", ([&]() {
std::string json_sequence_input = "\x1e[";
for (size_t i = 0; i < 2000; i++) {
json_sequence_input += '1';
json_sequence_input += (i + 1 < 2000 ? ',' : ']');
}
json_sequence_input += '\n';
auto input = simdjson::padded_string(json_sequence_input);
ASSERT_SUCCESS(parser.iterate_many(input, 1024, simdjson::stream_format::json_sequence).get(stream));
auto it = stream.begin();
ASSERT_ERROR(it.error(), CAPACITY);
return true;
}()));
TEST_SUCCEED();
}
bool comma_delimited_tests() {
TEST_START();
ondemand::parser parser;
ondemand::document_stream stream;
SUBTEST("basic comma-separated objects", ([&]() {
auto input = R"({"a":1},{"b":2},{"c":3})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
const char* keys[] = {"a", "b", "c"};
int64_t expected[] = {1, 2, 3};
size_t count = 0;
for (auto doc : stream) {
ASSERT_SUCCESS(doc.error());
int64_t value;
ASSERT_SUCCESS(doc[keys[count]].get(value));
ASSERT_EQUAL(value, expected[count]);
count++;
}
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("comma-separated with whitespace", ([&]() {
auto input = R"({"a":1} , {"b":2} , {"c":3})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("comma-separated arrays", ([&]() {
auto input = R"([1,2],[3,4],[5,6])"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(3));
return true;
}()));
SUBTEST("mixed document types", ([&]() {
auto input = R"({"a":1},[2,3],"string",42,true,null)"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(6));
return true;
}()));
SUBTEST("nested commas preserved", ([&]() {
auto input = R"({"a":[1,2,3]},{"b":{"c":4,"d":5}})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
// First doc: {"a":[1,2,3]}
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
ondemand::array arr;
ASSERT_SUCCESS(doc["a"].get_array().get(arr));
size_t arr_count = 0;
for (auto elem : arr) { (void)elem; arr_count++; }
ASSERT_EQUAL(arr_count, size_t(3));
}
++it;
// Second doc: {"b":{"c":4,"d":5}}
{
ondemand::document_reference doc;
ASSERT_SUCCESS((*it).get(doc));
ondemand::object b_obj;
ASSERT_SUCCESS(doc["b"].get_object().get(b_obj));
int64_t c_val, d_val;
ASSERT_SUCCESS(b_obj["c"].get(c_val));
ASSERT_SUCCESS(b_obj["d"].get(d_val));
ASSERT_EQUAL(c_val, int64_t(4));
ASSERT_EQUAL(d_val, int64_t(5));
}
return true;
}()));
SUBTEST("single document no comma", ([&]() {
auto input = R"({"a":1})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
size_t count = 0;
for (auto doc : stream) { ASSERT_SUCCESS(doc.error()); count++; }
ASSERT_EQUAL(count, size_t(1));
return true;
}()));
SUBTEST("current_index points to JSON value", ([&]() {
// Layout: {(0) "(1) a(2) "(3) :(4) 1(5) }(6) ,(7) {(8) ...
auto input = R"({"a":1},{"b":2})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(0)); // First { at position 0
++it;
ASSERT_EQUAL(it.current_index(), size_t(8)); // Second { at position 8
return true;
}()));
SUBTEST("source returns correct document slice", ([&]() {
auto input = R"([1,2,3],{"a":1})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, ondemand::DEFAULT_BATCH_SIZE, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.source(), std::string_view("[1,2,3]"));
++it;
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
return true;
}()));
SUBTEST("multibatch current_index and source", ([&]() {
auto input = R"({"a":1}, {"b":2})"_padded;
ASSERT_SUCCESS(parser.iterate_many(input, 10, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_EQUAL(it.current_index(), size_t(0));
ASSERT_EQUAL(it.source(), std::string_view("{\"a\":1}"));
++it;
ASSERT_EQUAL(it.current_index(), size_t(9));
ASSERT_EQUAL(it.source(), std::string_view("{\"b\":2}"));
return true;
}()));
SUBTEST("small batch reports capacity", ([&]() {
std::string comma_input = " [";
for (size_t i = 0; i < 2000; i++) {
comma_input += '1';
comma_input += (i + 1 < 2000 ? ',' : ']');
}
auto input = simdjson::padded_string(comma_input);
ASSERT_SUCCESS(parser.iterate_many(input, 1024, simdjson::stream_format::comma_delimited).get(stream));
auto it = stream.begin();
ASSERT_ERROR(it.error(), CAPACITY);
return true;
}()));
TEST_SUCCEED();
}
bool run() {
return
json_sequence_tests() &&
comma_delimited_tests() &&
issue2181() &&
issue2170() &&
issue2137() &&
@@ -988,6 +1295,7 @@ namespace document_stream_tests {
stress_data_race_with_error() &&
true;
}
} // document_stream_tests
int main (int argc, char *argv[]) {
tests/ondemand/ondemand_iterate_many_csv.cpp
+5 -5
View File
@@ -13,7 +13,7 @@ bool normal() {
auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), stream_format::comma_delimited).get(doc_stream));
for (auto doc : doc_stream)
{
@@ -28,7 +28,7 @@ bool small_batch_size() {
auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
ASSERT_SUCCESS(parser.iterate_many(json, 32, true).get(doc_stream));
ASSERT_SUCCESS(parser.iterate_many(json, 32, stream_format::comma_delimited).get(doc_stream));
for (auto doc : doc_stream)
{
@@ -43,7 +43,7 @@ bool trailing_comma() {
auto json = R"(1,)"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), stream_format::comma_delimited).get(doc_stream));
for (auto doc : doc_stream)
{
@@ -59,7 +59,7 @@ bool check_parsed_values() {
auto json = R"( 1 , "a" , [100, 1] , {"hello" : "world"} , )"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), stream_format::comma_delimited).get(doc_stream));
auto begin = doc_stream.begin();
auto end = doc_stream.end();
@@ -127,7 +127,7 @@ bool leading_comma() {
auto json = R"(,1)"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), true).get(doc_stream));
ASSERT_SUCCESS(parser.iterate_many(json, json.size(), stream_format::comma_delimited).get(doc_stream));
try {
auto begin = doc_stream.begin();
tests/ondemand/ondemand_readme_examples.cpp
+1 -4
View File
@@ -1706,10 +1706,7 @@ bool allow_comma_separated_example() {
auto json = R"( 1, 2, 3, 4, "a", "b", "c", {"hello": "world"} , [1, 2, 3])"_padded;
ondemand::parser parser;
ondemand::document_stream doc_stream;
// We pass '32' as the batch size, but it is a bogus parameter because, since
// we pass 'true' to the allow_comma parameter, the batch size will be set to at least
// the document size.
auto error = parser.iterate_many(json, 32, true).get(doc_stream);
auto error = parser.iterate_many(json, 32, simdjson::stream_format::comma_delimited).get(doc_stream);
if(error) { std::cerr << error << std::endl; return false; }
for (auto doc : doc_stream) {
std::cout << doc.type() << std::endl;