mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 12062bc5bc | |||
| 6e0abb7f9c | |||
| 686c9bff54 | |||
| e1654be9b3 | |||
| de1b38c459 | |||
| 3d08f995d0 | |||
| 135159012f | |||
| 936afe9aa1 | |||
| 1cc03882de | |||
| 9cd389ae84 | |||
| 64d5ee550f | |||
| 18cdecaf8e | |||
| c33e4441a0 | |||
| 732ca8cf70 | |||
| 2143ebe538 | |||
| df770af450 | |||
| 1985584de7 | |||
| 6a74cd41d6 | |||
| dcaad94b53 |
@@ -40,6 +40,7 @@ SIMDJSON_POP_DISABLE_WARNINGS
|
|||||||
#include "json2msgpack/boostjson.h"
|
#include "json2msgpack/boostjson.h"
|
||||||
|
|
||||||
#include "partial_tweets/simdjson_ondemand.h"
|
#include "partial_tweets/simdjson_ondemand.h"
|
||||||
|
#include "partial_tweets/simdjson_ondemand_key_selector.h"
|
||||||
#include "partial_tweets/simdjson_dom.h"
|
#include "partial_tweets/simdjson_dom.h"
|
||||||
#include "partial_tweets/yyjson.h"
|
#include "partial_tweets/yyjson.h"
|
||||||
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
#if SIMDJSON_COMPETITION_ONDEMAND_SAJSON
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
|
||||||
|
#include "partial_tweets.h"
|
||||||
|
|
||||||
|
namespace partial_tweets {
|
||||||
|
|
||||||
|
using namespace simdjson;
|
||||||
|
|
||||||
|
struct simdjson_ondemand_key_selector {
|
||||||
|
using StringType = std::string_view;
|
||||||
|
|
||||||
|
ondemand::parser parser{};
|
||||||
|
|
||||||
|
// Compile-time selectors — all PHF tables are static constexpr, so every
|
||||||
|
// call to match_raw fully inlines.
|
||||||
|
using tweet_sel_t = ondemand::key_selector<
|
||||||
|
"created_at", "id", "text", "in_reply_to_status_id",
|
||||||
|
"user", "retweet_count", "favorite_count">;
|
||||||
|
using user_sel_t = ondemand::key_selector<"id", "screen_name">;
|
||||||
|
|
||||||
|
simdjson_inline uint64_t nullable_int(ondemand::value value) {
|
||||||
|
if (value.is_null()) { return 0; }
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
simdjson_inline twitter_user<std::string_view> read_user(ondemand::object user) {
|
||||||
|
twitter_user<std::string_view> out{};
|
||||||
|
user.for_each<user_sel_t>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: out.id = uint64_t(v); break; // "id"
|
||||||
|
case 1: out.screen_name = std::string_view(v); break; // "screen_name"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
|
||||||
|
auto doc = parser.iterate(json);
|
||||||
|
for (ondemand::object tw : doc.find_field("statuses")) {
|
||||||
|
tweet<std::string_view> t{};
|
||||||
|
tw.for_each<tweet_sel_t>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: t.created_at = std::string_view(v); break; // "created_at"
|
||||||
|
case 1: t.id = uint64_t(v); break; // "id"
|
||||||
|
case 2: t.result = std::string_view(v); break; // "text"
|
||||||
|
case 3: t.in_reply_to_status_id = nullable_int(v); break; // "in_reply_to_status_id"
|
||||||
|
case 4: t.user = read_user(v); break; // "user"
|
||||||
|
case 5: t.retweet_count = uint64_t(v); break; // "retweet_count"
|
||||||
|
case 6: t.favorite_count = uint64_t(v); break; // "favorite_count"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
result.emplace_back(std::move(t));
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
BENCHMARK_TEMPLATE(partial_tweets, simdjson_ondemand_key_selector)->UseManualTime();
|
||||||
|
|
||||||
|
} // namespace partial_tweets
|
||||||
|
|
||||||
|
#endif // SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
@@ -61,22 +61,29 @@ struct yyjson_base {
|
|||||||
};
|
};
|
||||||
|
|
||||||
struct yyjson : yyjson_base {
|
struct yyjson : yyjson_base {
|
||||||
|
// The document owns the string memory that result's string_views point into,
|
||||||
|
// so it must outlive each run() (the verification diff happens after run()
|
||||||
|
// returns). Free it on the next run() / at destruction, not before the views
|
||||||
|
// are read.
|
||||||
|
yyjson_doc *doc{};
|
||||||
|
~yyjson() { if (doc != nullptr) { yyjson_doc_free(doc); } }
|
||||||
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
|
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
|
||||||
yyjson_doc *doc = yyjson_read(json.data(), json.size(), 0);
|
if (doc != nullptr) { yyjson_doc_free(doc); doc = nullptr; }
|
||||||
bool b = yyjson_base::run(doc, result);
|
doc = yyjson_read(json.data(), json.size(), 0);
|
||||||
yyjson_doc_free(doc);
|
return yyjson_base::run(doc, result);
|
||||||
return b;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
BENCHMARK_TEMPLATE(partial_tweets, yyjson)->UseManualTime();
|
BENCHMARK_TEMPLATE(partial_tweets, yyjson)->UseManualTime();
|
||||||
|
|
||||||
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
|
#if SIMDJSON_COMPETITION_ONDEMAND_INSITU
|
||||||
struct yyjson_insitu : yyjson_base {
|
struct yyjson_insitu : yyjson_base {
|
||||||
|
// See the note on yyjson above: the document must outlive result's views.
|
||||||
|
yyjson_doc *doc{};
|
||||||
|
~yyjson_insitu() { if (doc != nullptr) { yyjson_doc_free(doc); } }
|
||||||
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
|
bool run(simdjson::padded_string &json, std::vector<tweet<std::string_view>> &result) {
|
||||||
yyjson_doc *doc = yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0);
|
if (doc != nullptr) { yyjson_doc_free(doc); doc = nullptr; }
|
||||||
bool b = yyjson_base::run(doc, result);
|
doc = yyjson_read_opts(json.data(), json.size(), YYJSON_READ_INSITU, 0, 0);
|
||||||
yyjson_doc_free(doc);
|
return yyjson_base::run(doc, result);
|
||||||
return b;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
BENCHMARK_TEMPLATE(partial_tweets, yyjson_insitu)->UseManualTime();
|
BENCHMARK_TEMPLATE(partial_tweets, yyjson_insitu)->UseManualTime();
|
||||||
|
|||||||
+221
-3
@@ -25,6 +25,7 @@ separate document](https://github.com/simdjson/simdjson/blob/master/doc/builder.
|
|||||||
+ [Special cases](#special-cases)
|
+ [Special cases](#special-cases)
|
||||||
+ [Renaming and skipping fields with annotations](#renaming-and-skipping-fields-with-annotations)
|
+ [Renaming and skipping fields with annotations](#renaming-and-skipping-fields-with-annotations)
|
||||||
* [The simdjson::from shortcut (experimental, C++20)](#the-simdjsonfrom-shortcut-experimental-c20)
|
* [The simdjson::from shortcut (experimental, C++20)](#the-simdjsonfrom-shortcut-experimental-c20)
|
||||||
|
* [Order-independent reflective deserialization (experimental)](#order-independent-reflective-deserialization-experimental)
|
||||||
- [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
|
- [Minifying JSON strings without parsing](#minifying-json-strings-without-parsing)
|
||||||
- [UTF-8 validation (alone)](#utf-8-validation-alone)
|
- [UTF-8 validation (alone)](#utf-8-validation-alone)
|
||||||
- [JSON Pointer](#json-pointer)
|
- [JSON Pointer](#json-pointer)
|
||||||
@@ -32,6 +33,7 @@ separate document](https://github.com/simdjson/simdjson/blob/master/doc/builder.
|
|||||||
* [Using `for_each_at_path_with_wildcard` for JSONPath Queries (On-Demand)](#using-for_each_at_path_with_wildcard-for-jsonpath-queries-on-demand)
|
* [Using `for_each_at_path_with_wildcard` for JSONPath Queries (On-Demand)](#using-for_each_at_path_with_wildcard-for-jsonpath-queries-on-demand)
|
||||||
+ [Example Usage](#example-usage)
|
+ [Example Usage](#example-usage)
|
||||||
- [C++20 Ranges Support](#c20-ranges-support)
|
- [C++20 Ranges Support](#c20-ranges-support)
|
||||||
|
- [Key selectors](#key-selectors)
|
||||||
- [Compile-Time JSONPath and JSON Pointer (C++26 Reflection)](#compile-time-jsonpath-and-json-pointer-c26-reflection)
|
- [Compile-Time JSONPath and JSON Pointer (C++26 Reflection)](#compile-time-jsonpath-and-json-pointer-c26-reflection)
|
||||||
- [Error handling](#error-handling)
|
- [Error handling](#error-handling)
|
||||||
* [Error handling examples without exceptions](#error-handling-examples-without-exceptions)
|
* [Error handling examples without exceptions](#error-handling-examples-without-exceptions)
|
||||||
@@ -1446,7 +1448,9 @@ that are not made by Toyota.
|
|||||||
|
|
||||||
|
|
||||||
**Performance tip**: You will get better performance if you order the attributes (make, model)
|
**Performance tip**: You will get better performance if you order the attributes (make, model)
|
||||||
in the order they appear in the JSON document.
|
in the order they appear in the JSON document. Alternatively, [key selectors](#key-selectors)
|
||||||
|
let you extract a fixed, known set of fields in a single pass, independently of the
|
||||||
|
order in which they appear in the document.
|
||||||
|
|
||||||
|
|
||||||
### 3. Using static reflection (C++26)
|
### 3. Using static reflection (C++26)
|
||||||
@@ -1525,7 +1529,9 @@ void f() {
|
|||||||
|
|
||||||
|
|
||||||
**Performance tip**: You will get better performance if you order the attributes (make, model)
|
**Performance tip**: You will get better performance if you order the attributes (make, model)
|
||||||
in the order they appear in the JSON document.
|
in the order they appear in the JSON document. Alternatively, [key selectors](#key-selectors)
|
||||||
|
let you extract a fixed, known set of fields in a single pass, independently of the
|
||||||
|
order in which they appear in the document.
|
||||||
|
|
||||||
#### Special cases
|
#### Special cases
|
||||||
|
|
||||||
@@ -1678,6 +1684,56 @@ std::map<std::string, std::string> obj =
|
|||||||
|
|
||||||
The `simdjson::from` construction is EXPERIMENTAL and subject to changes.
|
The `simdjson::from` construction is EXPERIMENTAL and subject to changes.
|
||||||
|
|
||||||
|
### Order-independent reflective deserialization (experimental)
|
||||||
|
|
||||||
|
> **Experimental and opt-in.** This feature is disabled by default. Enable it by
|
||||||
|
> defining the macro `SIMDJSON_USE_KEY_SELECTOR_REFLECTION=1` before including
|
||||||
|
> simdjson (e.g. as a compiler flag `-DSIMDJSON_USE_KEY_SELECTOR_REFLECTION=1`).
|
||||||
|
> It requires C++26 static reflection (`SIMDJSON_STATIC_REFLECTION`).
|
||||||
|
|
||||||
|
By default, reflective deserialization (`doc.get<T>()` / `simdjson::from`) reads
|
||||||
|
each struct member with an ordered field lookup. This is fastest when the JSON
|
||||||
|
keys appear in the same order as the struct's members, which is the common case.
|
||||||
|
|
||||||
|
When you genuinely cannot rely on the JSON key order — for example when the data
|
||||||
|
comes from a producer that emits members in an arbitrary or varying order — the
|
||||||
|
ordered lookups degrade, because each out-of-order key forces a rescan of the
|
||||||
|
object. For that situation simdjson offers an optional, order-independent
|
||||||
|
strategy: when `SIMDJSON_USE_KEY_SELECTOR_REFLECTION=1` is defined, the
|
||||||
|
reflective deserializer builds a compile-time [key selector](#key-selectors)
|
||||||
|
from the struct's members and walks each object **once** with
|
||||||
|
`object::for_each`, classifying every key through a perfect hash regardless of
|
||||||
|
its position.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
// Compile this translation unit with -DSIMDJSON_USE_KEY_SELECTOR_REFLECTION=1
|
||||||
|
#include "simdjson.h"
|
||||||
|
using namespace simdjson;
|
||||||
|
|
||||||
|
struct Tweet {
|
||||||
|
uint64_t id;
|
||||||
|
std::string text;
|
||||||
|
uint64_t retweet_count;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Keys here are NOT in declaration order, yet deserialization succeeds.
|
||||||
|
auto json = R"({ "retweet_count": 7, "id": 12345, "text": "hello" })"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
Tweet t;
|
||||||
|
auto error = doc.get(t); // uses the key-selector path under the macro
|
||||||
|
```
|
||||||
|
|
||||||
|
**This is not a universal speedup — run your own benchmarks before adopting it.**
|
||||||
|
In our measurements the key-selector path is roughly on par with the default on
|
||||||
|
small structs (e.g. the Twitter user/tweet objects) but **slower** on documents
|
||||||
|
dominated by many small nested objects (e.g. the CITM catalog), because walking
|
||||||
|
every field of every object and hashing each key costs more than ordered,
|
||||||
|
short-circuiting lookups when the keys *are* in order. Only consider enabling the
|
||||||
|
macro when (a) your inputs really do present keys out of order, and (b) your own
|
||||||
|
benchmarks on your own data show a win. Otherwise, leave it off.
|
||||||
|
|
||||||
|
|
||||||
Minifying JSON strings without parsing
|
Minifying JSON strings without parsing
|
||||||
----------------------
|
----------------------
|
||||||
|
|
||||||
@@ -2010,6 +2066,168 @@ for (auto field_result : ondemand::get_key_value_range(obj)) {
|
|||||||
The range wrappers are zero-cost: they forward directly to the underlying
|
The range wrappers are zero-cost: they forward directly to the underlying
|
||||||
On-Demand iterators with no value buffering or extra per-element overhead.
|
On-Demand iterators with no value buffering or extra per-element overhead.
|
||||||
|
|
||||||
|
## Key selectors
|
||||||
|
|
||||||
|
> **Experimental.** Key selectors are an experimental feature: the API may change
|
||||||
|
> in a future release.
|
||||||
|
|
||||||
|
When you need to extract a known, fixed set of fields from a JSON object and you
|
||||||
|
do not care about the order in which they appear in the document, *key selectors*
|
||||||
|
let you do it in a single pass: each selected key is mapped to a small integer
|
||||||
|
index, and you dispatch on that index (typically with a `switch`) instead of
|
||||||
|
repeatedly looking up keys by string.
|
||||||
|
|
||||||
|
**Requirements.** Key selectors rely on C++20 features (concepts and class-type
|
||||||
|
non-type template parameters), so they are only available when simdjson is
|
||||||
|
compiled in C++20 mode or later. When that support is present, the macro
|
||||||
|
`SIMDJSON_SUPPORTS_CONCEPTS` is defined.
|
||||||
|
|
||||||
|
Key selection works with hashing.
|
||||||
|
A *hash function* maps keys (here, JSON field names such as `"id"` or `"name"`)
|
||||||
|
to small integers. A *perfect* hash function is one that, for a fixed and known
|
||||||
|
set of keys, maps each key to a distinct slot with no collisions, so a single
|
||||||
|
hash computation plus one comparison suffices to identify a key, there is no
|
||||||
|
probing and no collision chains.
|
||||||
|
|
||||||
|
Because the set of keys is known at compile time, the simdjson library builds the perfect
|
||||||
|
hash function during compilation (using `consteval`). All of its lookup tables
|
||||||
|
become `static constexpr` data, which the compiler treats as constants at every
|
||||||
|
call site and can fully inline. At run time, recognizing a field name reduces to:
|
||||||
|
scan its length, compute a hash from a couple of bytes, and perform one
|
||||||
|
length-and-bytes comparison, branch-light and SIMD-accelerated.
|
||||||
|
|
||||||
|
You declare a selector type from a list of string literals:
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
using sel = simdjson::ondemand::key_selector<"id", "name", "email">;
|
||||||
|
```
|
||||||
|
|
||||||
|
`sel` is a stateless type. Each key is assigned a fixed index, in declaration
|
||||||
|
order: `"id"` is 0, `"name"` is 1, `"email"` is 2. The type exposes a couple of
|
||||||
|
compile-time helpers:
|
||||||
|
|
||||||
|
- `sel::size()` — the number of keys in the selector (here, 3);
|
||||||
|
- `sel::key_at(i)` — the key text at index `i`.
|
||||||
|
|
||||||
|
You then walk an object with `object::for_each<sel>(callback)`. The callback is
|
||||||
|
invoked once for each field whose key belongs to the selector, **in JSON order**,
|
||||||
|
with two arguments: the selector index of the matched key, and the field's value
|
||||||
|
(an `ondemand::value` that you must consume inside the callback, before the next
|
||||||
|
field is visited, as usual with On Demand). Fields whose keys are not in the
|
||||||
|
selector are skipped without being parsed; duplicate keys are ignored after the
|
||||||
|
first match; and iteration stops as soon as every selector key has matched or the
|
||||||
|
object ends. The `for_each` call returns a `simdjson::error_code` (`SUCCESS`, or the first
|
||||||
|
error encountered while walking the object). Your callback may itself return a
|
||||||
|
`simdjson::error_code`: when it does, the walk stops at the first non-`SUCCESS`
|
||||||
|
result and `for_each` returns it. That is the recommended way to report a
|
||||||
|
value-parsing error (e.g. a type mismatch) from inside the callback.
|
||||||
|
|
||||||
|
Because the selector index is a small integer, a `switch` is the natural way to
|
||||||
|
dispatch on the matched field.
|
||||||
|
|
||||||
|
|
||||||
|
Key selectors are subject to a few compile-time restrictions:
|
||||||
|
|
||||||
|
- Key length. We currently limit keys to at most 31 characters long.
|
||||||
|
A longer key produces a compile-time error. This limitations could be
|
||||||
|
eased in the future but we expect longer keys to be unusual.
|
||||||
|
- Number of keys. The hard limit is 255 keys, but the compile-time perfect-hash
|
||||||
|
construction may fail (again, a compile-time error) for large or awkward key sets,
|
||||||
|
and compilation time grows with the number of keys. For compilation speed,
|
||||||
|
you may use precompiled headers if you have dozens of keys.
|
||||||
|
- Key contents. Keys must be distinct, non-empty, and must not contain a
|
||||||
|
backslash, a double quote, or a null byte. Matching is performed against the
|
||||||
|
raw, unescaped JSON key bytes, so a selector key has to equal the key exactly
|
||||||
|
as it appears in the document (no JSON escape processing is applied).
|
||||||
|
|
||||||
|
This example uses exceptions (see [Disabling exceptions](#disabling-exceptions)
|
||||||
|
for the error-code style). The `"age"` field is not selected, so it is skipped.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
using namespace simdjson;
|
||||||
|
auto json = R"({ "name": "Daniel", "age": 42, "city": "Montreal" })"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
ondemand::object obj = doc.get_object();
|
||||||
|
|
||||||
|
using fields = ondemand::key_selector<"name", "city">;
|
||||||
|
|
||||||
|
std::string_view name, city;
|
||||||
|
obj.for_each<fields>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: name = std::string_view(v); break; // "name"
|
||||||
|
case 1: city = std::string_view(v); break; // "city"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// name == "Daniel", city == "Montreal"
|
||||||
|
```
|
||||||
|
|
||||||
|
Key selectors work on any object and we do not have
|
||||||
|
to include all keys. Here the `"verified"` field is skipped.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
using namespace simdjson;
|
||||||
|
auto json = R"({ "user": { "id": 1186275104, "screen_name": "ayuu0123", "verified": false } })"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
ondemand::object user = doc["user"].get_object();
|
||||||
|
|
||||||
|
using user_fields = ondemand::key_selector<"id", "screen_name">;
|
||||||
|
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string_view handle;
|
||||||
|
user.for_each<user_fields>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: id = uint64_t(v); break; // "id"
|
||||||
|
case 1: handle = std::string_view(v); break; // "screen_name"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// id == 1186275104, handle == "ayuu0123"
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
A selected value can be any JSON value, including a nested object. Because the
|
||||||
|
value is consumed inside the callback, you can simply turn it into an
|
||||||
|
`ondemand::object` and call `for_each` again with another selector. As always
|
||||||
|
with On Demand, the inner object must be fully consumed before the outer
|
||||||
|
iteration continues, which the nested `for_each` does for you.
|
||||||
|
|
||||||
|
```cpp
|
||||||
|
using namespace simdjson;
|
||||||
|
auto json = R"({
|
||||||
|
"id": 42,
|
||||||
|
"author": { "name": "Daniel", "handle": "lemire" },
|
||||||
|
"title": "On Demand"
|
||||||
|
})"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
ondemand::object obj = doc.get_object();
|
||||||
|
|
||||||
|
using post_fields = ondemand::key_selector<"id", "author", "title">;
|
||||||
|
using author_fields = ondemand::key_selector<"name", "handle">;
|
||||||
|
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string_view title, author_name, author_handle;
|
||||||
|
obj.for_each<post_fields>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: id = uint64_t(v); break; // "id"
|
||||||
|
case 1: { // "author" is itself an object
|
||||||
|
ondemand::object author = v.get_object();
|
||||||
|
author.for_each<author_fields>([&](std::size_t j, ondemand::value av) {
|
||||||
|
switch (j) {
|
||||||
|
case 0: author_name = std::string_view(av); break; // "name"
|
||||||
|
case 1: author_handle = std::string_view(av); break; // "handle"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 2: title = std::string_view(v); break; // "title"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// id == 42, title == "On Demand", author_name == "Daniel", author_handle == "lemire"
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Compile-Time JSONPath and JSON Pointer (C++26 Reflection)
|
## Compile-Time JSONPath and JSON Pointer (C++26 Reflection)
|
||||||
|
|
||||||
The simdjson library provides **compile-time validated** JSONPath and JSON Pointer accessors when using C++26 Static Reflection. These accessors validate paths against struct definitions at compile time and generate optimized code with zero runtime overhead. In some cases, we find that it is much faster. Furthermore, it is safer in the sense that the expression
|
The simdjson library provides **compile-time validated** JSONPath and JSON Pointer accessors when using C++26 Static Reflection. These accessors validate paths against struct definitions at compile time and generate optimized code with zero runtime overhead. In some cases, we find that it is much faster. Furthermore, it is safer in the sense that the expression
|
||||||
@@ -3610,7 +3828,7 @@ Performance tips
|
|||||||
std::string_view year = data["year"];
|
std::string_view year = data["year"];
|
||||||
std::string_view rating = data["rating"];
|
std::string_view rating = data["rating"];
|
||||||
```
|
```
|
||||||
- You will get better performance if you seek the keys in the order in which they appear in the document. So if processing `{"a":1, "b":2, "c":3}`, do `value1 = data["a"]; value2 = data["b"]; value3 data["c"];` and not `value2 = data["b"]; value1 = data["a"]; value3 data["c"];`. Of course, it is not always possible to know for sure in which order the keys appear.
|
- You will get better performance if you seek the keys in the order in which they appear in the document. So if processing `{"a":1, "b":2, "c":3}`, do `value1 = data["a"]; value2 = data["b"]; value3 data["c"];` and not `value2 = data["b"]; value1 = data["a"]; value3 data["c"];`. Of course, it is not always possible to know for sure in which order the keys appear. See also [key selectors](#key-selectors), which extract a fixed, known set of fields in a single pass regardless of their order in the document.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
#include "simdjson/generic/ondemand/document.h"
|
#include "simdjson/generic/ondemand/document.h"
|
||||||
#include "simdjson/generic/ondemand/document_stream.h"
|
#include "simdjson/generic/ondemand/document_stream.h"
|
||||||
#include "simdjson/generic/ondemand/field.h"
|
#include "simdjson/generic/ondemand/field.h"
|
||||||
|
#include "simdjson/generic/ondemand/key_selector.h"
|
||||||
#include "simdjson/generic/ondemand/object.h"
|
#include "simdjson/generic/ondemand/object.h"
|
||||||
#include "simdjson/generic/ondemand/object_iterator.h"
|
#include "simdjson/generic/ondemand/object_iterator.h"
|
||||||
#include "simdjson/generic/ondemand/ranges.h"
|
#include "simdjson/generic/ondemand/ranges.h"
|
||||||
|
|||||||
@@ -10,6 +10,9 @@
|
|||||||
// Otherwise, amalgamation will fail.
|
// Otherwise, amalgamation will fail.
|
||||||
#include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY
|
#include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY
|
||||||
#include "simdjson/implementation.h"
|
#include "simdjson/implementation.h"
|
||||||
|
#include "simdjson/base.h"
|
||||||
|
#include "simdjson/common_defs.h"
|
||||||
|
#include "simdjson/constevalutil.h"
|
||||||
#include "simdjson/padded_string.h"
|
#include "simdjson/padded_string.h"
|
||||||
#include "simdjson/padded_string_view.h"
|
#include "simdjson/padded_string_view.h"
|
||||||
#include "simdjson/internal/dom_parser_implementation.h"
|
#include "simdjson/internal/dom_parser_implementation.h"
|
||||||
|
|||||||
@@ -0,0 +1,926 @@
|
|||||||
|
#ifndef SIMDJSON_GENERIC_ONDEMAND_KEY_SELECTOR_H
|
||||||
|
#define SIMDJSON_GENERIC_ONDEMAND_KEY_SELECTOR_H
|
||||||
|
|
||||||
|
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
|
||||||
|
#include "simdjson/base.h"
|
||||||
|
#include "simdjson/common_defs.h"
|
||||||
|
#include "simdjson/constevalutil.h"
|
||||||
|
#include "simdjson/generic/ondemand/raw_json_string.h"
|
||||||
|
#endif // SIMDJSON_CONDITIONAL_INCLUDE
|
||||||
|
|
||||||
|
#include <array>
|
||||||
|
#include <string_view>
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
#if defined(__aarch64__) || defined(__ARM_NEON)
|
||||||
|
#include <arm_neon.h>
|
||||||
|
#define SIMDJSON_KEY_SELECTOR_HAS_NEON 1
|
||||||
|
#else
|
||||||
|
#define SIMDJSON_KEY_SELECTOR_HAS_NEON 0
|
||||||
|
#endif
|
||||||
|
#if defined(__SSE2__)
|
||||||
|
#include <emmintrin.h>
|
||||||
|
#define SIMDJSON_KEY_SELECTOR_HAS_SSE2 1
|
||||||
|
#else
|
||||||
|
#define SIMDJSON_KEY_SELECTOR_HAS_SSE2 0
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
|
||||||
|
namespace simdjson {
|
||||||
|
namespace SIMDJSON_IMPLEMENTATION {
|
||||||
|
namespace ondemand {
|
||||||
|
|
||||||
|
namespace key_selector_detail {
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Compile-time perfect-hash generator.
|
||||||
|
// It scales to ~100 keys at compile time by determining association values one (position, character)
|
||||||
|
// symbol at a time (gperf-style) instead of an exhaustive offset search, and
|
||||||
|
// falls back to a Hash-and-Displace construction for large/awkward key sets.
|
||||||
|
//
|
||||||
|
// Only flat tables survive to runtime; the lookup is a few additions plus a
|
||||||
|
// single SIMD key comparison (see match_raw below).
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
// Maximum number of character positions the gperf hash may combine.
|
||||||
|
static constexpr std::size_t MAX_POSITIONS = 16;
|
||||||
|
// Sentinel "position" meaning "the last character of the key".
|
||||||
|
static constexpr std::size_t LAST_CHAR = std::size_t(-1);
|
||||||
|
// Runtime-encoded sentinels (stored in uint8 tables).
|
||||||
|
static constexpr std::uint8_t POS_LAST_CHAR = 0xFF; // positions_[i] == last char
|
||||||
|
static constexpr std::uint8_t HD_MODE = 0xFF; // num_positions == H&D mode
|
||||||
|
// Flags stored in positions[2] in H&D mode to select the key-hash variant.
|
||||||
|
static constexpr std::size_t HD_HASH_2BYTE_FLAG = 2;
|
||||||
|
static constexpr std::size_t HD_HASH_4BYTE_FLAG = 4;
|
||||||
|
|
||||||
|
constexpr std::size_t next_power_of_2(std::size_t n) noexcept {
|
||||||
|
if (n == 0) { return 1; }
|
||||||
|
std::size_t p = 1;
|
||||||
|
while (p < n) { p <<= 1; }
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Character at a given position (LAST_CHAR means last character), or 256 if out
|
||||||
|
// of bounds.
|
||||||
|
constexpr std::size_t char_at(std::string_view key, std::size_t pos) noexcept {
|
||||||
|
if (pos == LAST_CHAR) {
|
||||||
|
if (key.empty()) { return 256; }
|
||||||
|
return static_cast<unsigned char>(key[key.size() - 1]);
|
||||||
|
}
|
||||||
|
if (pos >= key.size()) { return 256; }
|
||||||
|
return static_cast<unsigned char>(key[pos]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count key pairs that a set of positions fails to distinguish. Keys whose
|
||||||
|
// lengths differ modulo the table size are separated by the length term in the
|
||||||
|
// hash, so they need no position coverage.
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval std::size_t count_undistinguished_pairs(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
const std::size_t* positions,
|
||||||
|
std::size_t num_positions,
|
||||||
|
std::size_t modulus) {
|
||||||
|
std::size_t count = 0;
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
for (std::size_t j = i + 1; j < N; ++j) {
|
||||||
|
if (keys[i].size() % modulus != keys[j].size() % modulus) { continue; }
|
||||||
|
bool distinguished = false;
|
||||||
|
for (std::size_t p = 0; p < num_positions; ++p) {
|
||||||
|
if (char_at(keys[i], positions[p]) != char_at(keys[j], positions[p])) {
|
||||||
|
distinguished = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!distinguished) { ++count; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval bool positions_distinguish(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
const std::size_t* positions,
|
||||||
|
std::size_t num_positions,
|
||||||
|
std::size_t modulus) {
|
||||||
|
return count_undistinguished_pairs<N>(keys, positions, num_positions, modulus) == 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Number of distinct (length % modulus, char_at(key, pos)) pairs at a position.
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval std::size_t discriminating_power(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
std::size_t pos,
|
||||||
|
std::size_t modulus) {
|
||||||
|
struct pair { std::size_t len_mod; std::size_t ch; };
|
||||||
|
std::array<pair, N> pairs{};
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
pairs[i] = {keys[i].size() % modulus, char_at(keys[i], pos)};
|
||||||
|
}
|
||||||
|
std::size_t count = 0;
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
bool dup = false;
|
||||||
|
for (std::size_t j = 0; j < i; ++j) {
|
||||||
|
if (pairs[i].len_mod == pairs[j].len_mod && pairs[i].ch == pairs[j].ch) {
|
||||||
|
dup = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!dup) { ++count; }
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval std::size_t max_key_length(const std::array<std::string_view, N>& keys) {
|
||||||
|
std::size_t m = 0;
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
if (keys[i].size() > m) { m = keys[i].size(); }
|
||||||
|
}
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bounded backtracking DFS for a minimal set of distinguishing positions.
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval bool backtracking_search(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
const std::size_t* candidates,
|
||||||
|
std::size_t num_candidates,
|
||||||
|
std::size_t* positions,
|
||||||
|
std::size_t& num_positions_out,
|
||||||
|
std::size_t& budget,
|
||||||
|
std::size_t modulus) {
|
||||||
|
constexpr std::size_t MAX_DEPTH = 8;
|
||||||
|
std::size_t breadth = num_candidates < 20 ? num_candidates : 20;
|
||||||
|
|
||||||
|
struct frame { std::size_t depth; std::size_t next_ci; std::size_t parent_count; };
|
||||||
|
std::array<frame, MAX_DEPTH + 1> stack{};
|
||||||
|
std::size_t sp = 0;
|
||||||
|
|
||||||
|
std::size_t initial_count = count_undistinguished_pairs<N>(keys, positions, 0, modulus);
|
||||||
|
if (budget > 0) { --budget; }
|
||||||
|
if (initial_count == 0) { num_positions_out = 0; return true; }
|
||||||
|
|
||||||
|
stack[0] = {0, 0, initial_count};
|
||||||
|
|
||||||
|
while (budget > 0) {
|
||||||
|
if (sp > MAX_DEPTH) {
|
||||||
|
if (sp == 0) { break; }
|
||||||
|
--sp;
|
||||||
|
++stack[sp].next_ci;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
auto& f = stack[sp];
|
||||||
|
if (f.next_ci >= breadth) {
|
||||||
|
if (sp == 0) { break; }
|
||||||
|
--sp;
|
||||||
|
++stack[sp].next_ci;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
positions[sp] = candidates[f.next_ci];
|
||||||
|
--budget;
|
||||||
|
std::size_t new_count = count_undistinguished_pairs<N>(keys, positions, sp + 1, modulus);
|
||||||
|
if (new_count == 0) { num_positions_out = sp + 1; return true; }
|
||||||
|
if (new_count < f.parent_count && sp + 1 < MAX_DEPTH) {
|
||||||
|
stack[sp + 1] = {sp + 1, f.next_ci + 1, new_count};
|
||||||
|
++sp;
|
||||||
|
} else {
|
||||||
|
++f.next_ci;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Phase 1: select character positions that distinguish all colliding pairs.
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval std::size_t select_positions(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
std::array<std::size_t, MAX_POSITIONS>& positions,
|
||||||
|
std::size_t modulus) {
|
||||||
|
if (positions_distinguish<N>(keys, positions.data(), 0, modulus)) { return 0; }
|
||||||
|
|
||||||
|
std::size_t max_len = max_key_length(keys);
|
||||||
|
constexpr std::size_t MAX_CANDIDATES = 256;
|
||||||
|
std::array<std::size_t, MAX_CANDIDATES> candidates{};
|
||||||
|
std::array<std::size_t, MAX_CANDIDATES> powers{};
|
||||||
|
std::size_t num_candidates = 0;
|
||||||
|
for (std::size_t p = 0; p < max_len && num_candidates < MAX_CANDIDATES - 1; ++p) {
|
||||||
|
candidates[num_candidates] = p;
|
||||||
|
powers[num_candidates] = discriminating_power(keys, p, modulus);
|
||||||
|
++num_candidates;
|
||||||
|
}
|
||||||
|
if (num_candidates < MAX_CANDIDATES) {
|
||||||
|
candidates[num_candidates] = LAST_CHAR;
|
||||||
|
powers[num_candidates] = discriminating_power(keys, LAST_CHAR, modulus);
|
||||||
|
++num_candidates;
|
||||||
|
}
|
||||||
|
for (std::size_t i = 0; i < num_candidates; ++i) {
|
||||||
|
for (std::size_t j = i + 1; j < num_candidates; ++j) {
|
||||||
|
if (powers[j] > powers[i]) {
|
||||||
|
auto tc = candidates[i]; candidates[i] = candidates[j]; candidates[j] = tc;
|
||||||
|
auto tp = powers[i]; powers[i] = powers[j]; powers[j] = tp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
positions[0] = candidates[0];
|
||||||
|
if (positions_distinguish<N>(keys, positions.data(), 1, modulus)) { return 1; }
|
||||||
|
|
||||||
|
positions[0] = 0;
|
||||||
|
positions[1] = LAST_CHAR;
|
||||||
|
if (positions_distinguish<N>(keys, positions.data(), 2, modulus)) { return 2; }
|
||||||
|
|
||||||
|
{
|
||||||
|
std::size_t budget = 5000;
|
||||||
|
std::size_t num_found = 0;
|
||||||
|
if (backtracking_search<N>(keys, candidates.data(), num_candidates,
|
||||||
|
positions.data(), num_found, budget, modulus)) {
|
||||||
|
return num_found;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::size_t num_pos = 0;
|
||||||
|
for (std::size_t ci = 0; ci < num_candidates && num_pos < MAX_POSITIONS; ++ci) {
|
||||||
|
bool already = false;
|
||||||
|
for (std::size_t p = 0; p < num_pos; ++p) {
|
||||||
|
if (positions[p] == candidates[ci]) { already = true; break; }
|
||||||
|
}
|
||||||
|
if (already) { continue; }
|
||||||
|
positions[num_pos] = candidates[ci];
|
||||||
|
++num_pos;
|
||||||
|
if (positions_distinguish<N>(keys, positions.data(), num_pos, modulus)) { return num_pos; }
|
||||||
|
}
|
||||||
|
|
||||||
|
throw "Failed to find distinguishing positions for perfect hash";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result of PHF computation. A max-sized slot_to_key array lets the same struct
|
||||||
|
// type carry any chosen table size.
|
||||||
|
template <std::size_t N>
|
||||||
|
struct phf_result {
|
||||||
|
// Allow up to 8x the minimum table size. Sparser tables solve faster.
|
||||||
|
static constexpr std::size_t MAX_TABLE_SIZE = next_power_of_2(N) * 8;
|
||||||
|
std::size_t table_size{};
|
||||||
|
std::array<std::array<std::size_t, 256>, MAX_POSITIONS> asso_values{};
|
||||||
|
std::size_t num_positions{};
|
||||||
|
std::array<std::size_t, MAX_POSITIONS> positions{};
|
||||||
|
std::array<std::size_t, MAX_TABLE_SIZE> slot_to_key{};
|
||||||
|
};
|
||||||
|
|
||||||
|
// Partition-based asso_values search (gperf-style). Determines asso_values one
|
||||||
|
// (position, character) symbol at a time; never revisits a value. Equivalence
|
||||||
|
// classes (keys sharing the same undetermined symbols) keep the search cheap.
|
||||||
|
template <std::size_t N, std::size_t M>
|
||||||
|
consteval bool try_generate_gperf(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
std::array<std::array<std::size_t, 256>, MAX_POSITIONS>& asso_values,
|
||||||
|
std::size_t& num_positions,
|
||||||
|
std::array<std::size_t, MAX_POSITIONS>& positions,
|
||||||
|
std::array<std::size_t, M>& slot_to_key) {
|
||||||
|
num_positions = select_positions<N>(keys, positions, M);
|
||||||
|
|
||||||
|
for (std::size_t p = 0; p < MAX_POSITIONS; ++p) {
|
||||||
|
for (std::size_t c = 0; c < 256; ++c) { asso_values[p][c] = 0; }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (num_positions == 0) {
|
||||||
|
for (std::size_t i = 0; i < M; ++i) { slot_to_key[i] = N; }
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
std::size_t slot = keys[i].size() % M;
|
||||||
|
if (slot_to_key[slot] != N) { return false; }
|
||||||
|
slot_to_key[slot] = i;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::array<std::array<std::size_t, MAX_POSITIONS>, N> kchars{};
|
||||||
|
for (std::size_t k = 0; k < N; ++k) {
|
||||||
|
for (std::size_t p = 0; p < num_positions; ++p) {
|
||||||
|
kchars[k][p] = char_at(keys[k], positions[p]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct sym_t { std::size_t pos; std::size_t ch; std::size_t freq; };
|
||||||
|
constexpr std::size_t MAX_SYMS = MAX_POSITIONS * 256;
|
||||||
|
std::array<sym_t, MAX_SYMS> syms{};
|
||||||
|
std::size_t nsyms = 0;
|
||||||
|
for (std::size_t p = 0; p < num_positions; ++p) {
|
||||||
|
std::array<std::size_t, 256> freq{};
|
||||||
|
for (std::size_t k = 0; k < N; ++k) {
|
||||||
|
std::size_t c = kchars[k][p];
|
||||||
|
if (c < 256) { freq[c]++; }
|
||||||
|
}
|
||||||
|
for (std::size_t c = 0; c < 256; ++c) {
|
||||||
|
if (freq[c] > 0) { syms[nsyms++] = {p, c, freq[c]}; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (std::size_t i = 0; i < nsyms; ++i) {
|
||||||
|
for (std::size_t j = i + 1; j < nsyms; ++j) {
|
||||||
|
if (syms[j].freq > syms[i].freq) {
|
||||||
|
auto tmp = syms[i]; syms[i] = syms[j]; syms[j] = tmp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
std::array<std::size_t, N> phash{};
|
||||||
|
for (std::size_t k = 0; k < N; ++k) { phash[k] = keys[k].size(); }
|
||||||
|
|
||||||
|
// Equivalence-class signatures: sig[k] = XOR of a per-symbol salt over the
|
||||||
|
// key's undetermined symbols. Updated incrementally as symbols are fixed.
|
||||||
|
std::array<std::array<std::size_t, 256>, MAX_POSITIONS> salt{};
|
||||||
|
{
|
||||||
|
std::size_t s = 0x9e3779b97f4a7c15ULL;
|
||||||
|
for (std::size_t p = 0; p < num_positions; ++p) {
|
||||||
|
for (std::size_t c = 0; c < 256; ++c) {
|
||||||
|
s = s * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||||
|
salt[p][c] = s;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::array<std::size_t, N> sig{};
|
||||||
|
for (std::size_t k = 0; k < N; ++k) {
|
||||||
|
std::size_t s = 0;
|
||||||
|
for (std::size_t p = 0; p < num_positions; ++p) {
|
||||||
|
std::size_t c = kchars[k][p];
|
||||||
|
if (c < 256) { s ^= salt[p][c]; }
|
||||||
|
}
|
||||||
|
sig[k] = s;
|
||||||
|
}
|
||||||
|
std::array<std::size_t, N> order{};
|
||||||
|
for (std::size_t k = 0; k < N; ++k) { order[k] = k; }
|
||||||
|
|
||||||
|
std::array<std::size_t, M> slot_gen{};
|
||||||
|
std::size_t gen = 0;
|
||||||
|
|
||||||
|
std::size_t search_limit = next_power_of_2(M);
|
||||||
|
if (search_limit < 32) { search_limit = 32; }
|
||||||
|
|
||||||
|
for (std::size_t si = 0; si < nsyms; ++si) {
|
||||||
|
std::size_t sp = syms[si].pos;
|
||||||
|
std::size_t sc = syms[si].ch;
|
||||||
|
|
||||||
|
std::size_t sp_salt = salt[sp][sc];
|
||||||
|
for (std::size_t k = 0; k < N; ++k) {
|
||||||
|
if (kchars[k][sp] == sc) { sig[k] ^= sp_salt; }
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::size_t i = 1; i < N; ++i) {
|
||||||
|
std::size_t x = order[i];
|
||||||
|
std::size_t xs = sig[x];
|
||||||
|
std::size_t j = i;
|
||||||
|
while (j > 0 && sig[order[j - 1]] > xs) {
|
||||||
|
order[j] = order[j - 1];
|
||||||
|
--j;
|
||||||
|
}
|
||||||
|
order[j] = x;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool found = false;
|
||||||
|
for (std::size_t v = 0; v < search_limit && !found; ++v) {
|
||||||
|
bool collision = false;
|
||||||
|
std::size_t ci = 0;
|
||||||
|
while (ci < N && !collision) {
|
||||||
|
std::size_t class_sig = sig[order[ci]];
|
||||||
|
std::size_t cj = ci;
|
||||||
|
while (cj < N && sig[order[cj]] == class_sig) { ++cj; }
|
||||||
|
if (cj - ci > 1) {
|
||||||
|
++gen;
|
||||||
|
for (std::size_t x = ci; x < cj; ++x) {
|
||||||
|
std::size_t k = order[x];
|
||||||
|
std::size_t h = phash[k];
|
||||||
|
if (kchars[k][sp] == sc) { h += v; }
|
||||||
|
h %= M;
|
||||||
|
if (slot_gen[h] == gen) { collision = true; break; }
|
||||||
|
slot_gen[h] = gen;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ci = cj;
|
||||||
|
}
|
||||||
|
if (!collision) {
|
||||||
|
asso_values[sp][sc] = v;
|
||||||
|
for (std::size_t k = 0; k < N; ++k) {
|
||||||
|
if (kchars[k][sp] == sc) { phash[k] += v; }
|
||||||
|
}
|
||||||
|
found = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!found) { return false; }
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::size_t i = 0; i < M; ++i) { slot_to_key[i] = N; }
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
std::size_t slot = phash[i] % M;
|
||||||
|
if (slot_to_key[slot] != N) { return false; }
|
||||||
|
slot_to_key[slot] = i;
|
||||||
|
}
|
||||||
|
std::size_t filled = 0;
|
||||||
|
for (std::size_t i = 0; i < M; ++i) {
|
||||||
|
if (slot_to_key[i] != N) { ++filled; }
|
||||||
|
}
|
||||||
|
return filled == N;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N, std::size_t M>
|
||||||
|
consteval bool try_compute_phf(const std::array<std::string_view, N>& keys, phf_result<N>& result) {
|
||||||
|
static_assert(M <= phf_result<N>::MAX_TABLE_SIZE, "Table size M exceeds maximum");
|
||||||
|
std::array<std::array<std::size_t, 256>, MAX_POSITIONS> asso{};
|
||||||
|
std::size_t npos{};
|
||||||
|
std::array<std::size_t, MAX_POSITIONS> pos{};
|
||||||
|
std::array<std::size_t, M> s2k{};
|
||||||
|
if (try_generate_gperf<N, M>(keys, asso, npos, pos, s2k)) {
|
||||||
|
result.table_size = M;
|
||||||
|
result.asso_values = asso;
|
||||||
|
result.num_positions = npos;
|
||||||
|
result.positions = pos;
|
||||||
|
for (std::size_t i = 0; i < M; ++i) { result.slot_to_key[i] = s2k[i]; }
|
||||||
|
for (std::size_t i = M; i < phf_result<N>::MAX_TABLE_SIZE; ++i) { result.slot_to_key[i] = N; }
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N, std::size_t M, std::size_t MaxM>
|
||||||
|
consteval bool try_gperf_po2(const std::array<std::string_view, N>& keys, phf_result<N>& result) {
|
||||||
|
if (try_compute_phf<N, M>(keys, result)) { return true; }
|
||||||
|
constexpr std::size_t NextM = M * 2;
|
||||||
|
if constexpr (NextM <= MaxM) { return try_gperf_po2<N, NextM, MaxM>(keys, result); }
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Hash-and-Displace fallback --------------------------------------------
|
||||||
|
|
||||||
|
constexpr std::size_t hd_bucket_hash(std::string_view key) noexcept {
|
||||||
|
std::size_t c0 = key.empty() ? 0 : static_cast<unsigned char>(key[0]);
|
||||||
|
std::size_t c1 = key.empty() ? 0 : static_cast<unsigned char>(key[key.size() - 1]);
|
||||||
|
return (c0 + c1 * 3 + key.size() * 17) & 0xFF;
|
||||||
|
}
|
||||||
|
constexpr std::size_t hd_safe_char(const char* p, std::size_t len, std::size_t idx) noexcept {
|
||||||
|
std::size_t has = static_cast<std::size_t>(idx < len);
|
||||||
|
std::size_t si = idx & (std::size_t{0} - has);
|
||||||
|
return static_cast<unsigned char>(p[si]) & (std::size_t{0} - has);
|
||||||
|
}
|
||||||
|
constexpr std::size_t hd_key_hash_2(std::string_view key) noexcept {
|
||||||
|
std::size_t kc = key.size();
|
||||||
|
kc = kc * 31 + static_cast<unsigned char>(key[0]);
|
||||||
|
kc = kc * 31 + hd_safe_char(key.data(), key.size(), 1);
|
||||||
|
return kc;
|
||||||
|
}
|
||||||
|
constexpr std::size_t hd_key_hash_4(std::string_view key) noexcept {
|
||||||
|
std::size_t kc = key.size();
|
||||||
|
kc = kc * 31 + static_cast<unsigned char>(key[0]);
|
||||||
|
kc = kc * 31 + hd_safe_char(key.data(), key.size(), 1);
|
||||||
|
kc = kc * 31 + hd_safe_char(key.data(), key.size(), 2);
|
||||||
|
kc = kc * 31 + hd_safe_char(key.data(), key.size(), 3);
|
||||||
|
return kc;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N, std::size_t M>
|
||||||
|
consteval bool try_hash_and_displace(
|
||||||
|
const std::array<std::string_view, N>& keys,
|
||||||
|
std::array<std::array<std::size_t, 256>, MAX_POSITIONS>& asso_values,
|
||||||
|
std::size_t& num_positions,
|
||||||
|
std::array<std::size_t, MAX_POSITIONS>& positions,
|
||||||
|
std::array<std::size_t, M>& slot_to_key) {
|
||||||
|
num_positions = select_positions<N>(keys, positions, M);
|
||||||
|
|
||||||
|
if (num_positions == 0) {
|
||||||
|
for (std::size_t i = 0; i < 256; ++i) { asso_values[0][i] = 0; }
|
||||||
|
for (std::size_t i = 0; i < M; ++i) { slot_to_key[i] = N; }
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
std::size_t slot = keys[i].size() % M;
|
||||||
|
if (slot_to_key[slot] != N) { return false; }
|
||||||
|
slot_to_key[slot] = i;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::size_t i = 0; i < 256; ++i) { asso_values[0][i] = 0; }
|
||||||
|
num_positions = HD_MODE; // sentinel for H&D mode
|
||||||
|
positions[0] = 0;
|
||||||
|
positions[1] = LAST_CHAR;
|
||||||
|
|
||||||
|
std::array<std::size_t, N> key_bucket{};
|
||||||
|
for (std::size_t i = 0; i < N; ++i) { key_bucket[i] = hd_bucket_hash(keys[i]); }
|
||||||
|
|
||||||
|
struct bucket_info { std::size_t ch; std::size_t count; };
|
||||||
|
std::array<bucket_info, N> buckets{};
|
||||||
|
std::size_t num_buckets = 0;
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
std::size_t bk = key_bucket[i];
|
||||||
|
bool found = false;
|
||||||
|
for (std::size_t b = 0; b < num_buckets; ++b) {
|
||||||
|
if (buckets[b].ch == bk) { ++buckets[b].count; found = true; break; }
|
||||||
|
}
|
||||||
|
if (!found) { buckets[num_buckets++] = {bk, 1}; }
|
||||||
|
}
|
||||||
|
for (std::size_t i = 0; i < num_buckets; ++i) {
|
||||||
|
for (std::size_t j = i + 1; j < num_buckets; ++j) {
|
||||||
|
if (buckets[j].count > buckets[i].count) {
|
||||||
|
auto tmp = buckets[i]; buckets[i] = buckets[j]; buckets[j] = tmp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
auto try_placement = [&](auto key_hash_fn) -> bool {
|
||||||
|
for (std::size_t i = 0; i < M; ++i) { slot_to_key[i] = N; }
|
||||||
|
for (std::size_t i = 0; i < 256; ++i) { asso_values[0][i] = 0; }
|
||||||
|
for (std::size_t b = 0; b < num_buckets; ++b) {
|
||||||
|
std::size_t ch = buckets[b].ch;
|
||||||
|
std::array<std::size_t, N> bucket_keys{};
|
||||||
|
std::size_t bk_count = 0;
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
if (key_bucket[i] == ch) { bucket_keys[bk_count++] = i; }
|
||||||
|
}
|
||||||
|
bool placed = false;
|
||||||
|
std::size_t max_d = M < 255 ? M : 255;
|
||||||
|
for (std::size_t d = 0; d < max_d; ++d) {
|
||||||
|
bool ok = true;
|
||||||
|
std::array<std::size_t, N> bucket_slots{};
|
||||||
|
for (std::size_t k = 0; k < bk_count; ++k) {
|
||||||
|
std::size_t slot = (d + key_hash_fn(keys[bucket_keys[k]])) % M;
|
||||||
|
if (slot_to_key[slot] != N) { ok = false; break; }
|
||||||
|
for (std::size_t k2 = 0; k2 < k; ++k2) {
|
||||||
|
if (bucket_slots[k2] == slot) { ok = false; break; }
|
||||||
|
}
|
||||||
|
if (!ok) { break; }
|
||||||
|
bucket_slots[k] = slot;
|
||||||
|
}
|
||||||
|
if (ok) {
|
||||||
|
asso_values[0][ch] = d;
|
||||||
|
for (std::size_t k = 0; k < bk_count; ++k) {
|
||||||
|
slot_to_key[bucket_slots[k]] = bucket_keys[k];
|
||||||
|
}
|
||||||
|
placed = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!placed) { return false; }
|
||||||
|
}
|
||||||
|
std::size_t filled = 0;
|
||||||
|
for (std::size_t i = 0; i < M; ++i) {
|
||||||
|
if (slot_to_key[i] != N) { ++filled; }
|
||||||
|
}
|
||||||
|
return filled == N;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (try_placement([](std::string_view k) { return hd_key_hash_2(k); })) {
|
||||||
|
positions[2] = HD_HASH_2BYTE_FLAG;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (try_placement([](std::string_view k) { return hd_key_hash_4(k); })) {
|
||||||
|
positions[2] = HD_HASH_4BYTE_FLAG;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N, std::size_t M>
|
||||||
|
consteval bool try_compute_phf_hd(const std::array<std::string_view, N>& keys, phf_result<N>& result) {
|
||||||
|
static_assert(M <= phf_result<N>::MAX_TABLE_SIZE, "Table size M exceeds maximum");
|
||||||
|
std::array<std::array<std::size_t, 256>, MAX_POSITIONS> asso{};
|
||||||
|
std::size_t npos{};
|
||||||
|
std::array<std::size_t, MAX_POSITIONS> pos{};
|
||||||
|
std::array<std::size_t, M> s2k{};
|
||||||
|
if (try_hash_and_displace<N, M>(keys, asso, npos, pos, s2k)) {
|
||||||
|
result.table_size = M;
|
||||||
|
result.asso_values = asso;
|
||||||
|
result.num_positions = npos;
|
||||||
|
result.positions = pos;
|
||||||
|
for (std::size_t i = 0; i < M; ++i) { result.slot_to_key[i] = s2k[i]; }
|
||||||
|
for (std::size_t i = M; i < phf_result<N>::MAX_TABLE_SIZE; ++i) { result.slot_to_key[i] = N; }
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <std::size_t N, std::size_t M>
|
||||||
|
consteval phf_result<N> compute_phf_hd_po2(const std::array<std::string_view, N>& keys) {
|
||||||
|
static_assert(M <= phf_result<N>::MAX_TABLE_SIZE, "Table size M exceeds maximum");
|
||||||
|
phf_result<N> result{};
|
||||||
|
if (try_compute_phf_hd<N, M>(keys, result)) { return result; }
|
||||||
|
constexpr std::size_t NextM = M * 2;
|
||||||
|
if constexpr (NextM <= phf_result<N>::MAX_TABLE_SIZE) {
|
||||||
|
return compute_phf_hd_po2<N, NextM>(keys);
|
||||||
|
} else {
|
||||||
|
throw "Hash-and-Displace: failed to find valid table size";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute a perfect hash for `keys`: try gperf at power-of-two sizes (capped so
|
||||||
|
// the runtime tables stay within uint8 indices), then fall back to H&D.
|
||||||
|
template <std::size_t N>
|
||||||
|
consteval phf_result<N> compute_phf(const std::array<std::string_view, N>& keys) {
|
||||||
|
constexpr std::size_t StartM = next_power_of_2(N);
|
||||||
|
constexpr std::size_t GPERF_MAX_TABLE =
|
||||||
|
phf_result<N>::MAX_TABLE_SIZE < 256 ? phf_result<N>::MAX_TABLE_SIZE : 256;
|
||||||
|
if constexpr (StartM <= GPERF_MAX_TABLE) {
|
||||||
|
phf_result<N> result{};
|
||||||
|
if (try_gperf_po2<N, StartM, GPERF_MAX_TABLE>(keys, result)) { return result; }
|
||||||
|
}
|
||||||
|
return compute_phf_hd_po2<N, StartM>(keys);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// Runtime tables (flat, uint8) derived from a phf_result.
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
template <std::size_t N, std::size_t TableSize, std::size_t MaxKeyLen>
|
||||||
|
struct phf_data {
|
||||||
|
std::array<std::array<std::uint8_t, 256>, MAX_POSITIONS> asso_values{};
|
||||||
|
std::array<std::uint8_t, MAX_POSITIONS> positions{};
|
||||||
|
std::uint8_t num_positions{};
|
||||||
|
std::uint8_t hd_hash_variant{}; // 2 or 4 (H&D only)
|
||||||
|
std::array<std::uint8_t, TableSize> slot_to_key{};
|
||||||
|
// slot_key_bytes[s] holds the key stored at slot s, zero-padded to a 16-byte
|
||||||
|
// multiple so the SIMD comparison can read a whole register.
|
||||||
|
std::array<std::array<char, ((MaxKeyLen + 15) / 16) * 16>, TableSize> slot_key_bytes{};
|
||||||
|
std::array<std::uint8_t, TableSize> slot_key_len{};
|
||||||
|
};
|
||||||
|
|
||||||
|
template <std::size_t N>
|
||||||
|
constexpr std::size_t compute_max_key_len(const std::array<std::string_view, N>& keys) noexcept {
|
||||||
|
std::size_t m = 0;
|
||||||
|
for (std::size_t i = 0; i < N; ++i) { if (keys[i].size() > m) { m = keys[i].size(); } }
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate keys and build the runtime tables from the computed perfect hash.
|
||||||
|
template <std::size_t N, std::size_t TableSize, std::size_t MaxKeyLen>
|
||||||
|
consteval phf_data<N, TableSize, MaxKeyLen>
|
||||||
|
build_phf_data(const std::array<std::string_view, N>& keys, const phf_result<N>& result) {
|
||||||
|
for (std::size_t i = 0; i < N; ++i) {
|
||||||
|
if (keys[i].empty()) { throw "empty keys are not allowed in key_selector"; }
|
||||||
|
if (keys[i].size() > MaxKeyLen) { throw "key length exceeds MaxKeyLen"; }
|
||||||
|
for (char c : keys[i]) {
|
||||||
|
if (c == '\\') { throw "backslash not allowed in key_selector keys"; }
|
||||||
|
if (c == '"') { throw "quote not allowed in key_selector keys"; }
|
||||||
|
if (c == '\0') { throw "null byte not allowed in key_selector keys"; }
|
||||||
|
}
|
||||||
|
for (std::size_t j = i + 1; j < N; ++j) {
|
||||||
|
if (keys[i] == keys[j]) { throw "duplicate keys in key_selector"; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
phf_data<N, TableSize, MaxKeyLen> out{};
|
||||||
|
|
||||||
|
if (result.num_positions == HD_MODE) {
|
||||||
|
// H&D mode: single displacement table in asso_values[0].
|
||||||
|
for (std::size_t c = 0; c < 256; ++c) {
|
||||||
|
out.asso_values[0][c] = static_cast<std::uint8_t>(result.asso_values[0][c]);
|
||||||
|
}
|
||||||
|
out.num_positions = static_cast<std::uint8_t>(HD_MODE);
|
||||||
|
out.hd_hash_variant = static_cast<std::uint8_t>(result.positions[2]);
|
||||||
|
} else {
|
||||||
|
for (std::size_t pi = 0; pi < result.num_positions; ++pi) {
|
||||||
|
for (std::size_t c = 0; c < 256; ++c) {
|
||||||
|
out.asso_values[pi][c] = static_cast<std::uint8_t>(result.asso_values[pi][c] % TableSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.num_positions = static_cast<std::uint8_t>(result.num_positions);
|
||||||
|
for (std::size_t i = 0; i < result.num_positions; ++i) {
|
||||||
|
out.positions[i] = (result.positions[i] == LAST_CHAR)
|
||||||
|
? POS_LAST_CHAR
|
||||||
|
: static_cast<std::uint8_t>(result.positions[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::size_t s = 0; s < TableSize; ++s) {
|
||||||
|
out.slot_to_key[s] = static_cast<std::uint8_t>(result.slot_to_key[s]);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (std::size_t s = 0; s < TableSize; ++s) {
|
||||||
|
std::size_t ki = result.slot_to_key[s];
|
||||||
|
if (ki < N) {
|
||||||
|
auto k = keys[ki];
|
||||||
|
out.slot_key_len[s] = static_cast<std::uint8_t>(k.size());
|
||||||
|
for (std::size_t c = 0; c < k.size(); ++c) { out.slot_key_bytes[s][c] = k[c]; }
|
||||||
|
} else {
|
||||||
|
out.slot_key_len[s] = 0; // empty slot: no length can match
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- SIMD runtime primitives ------------------------------------------------
|
||||||
|
|
||||||
|
// Scan for the terminating '"' starting at p. Returns its byte offset (= key
|
||||||
|
// length). Caller guarantees SIMDJSON_PADDING bytes past the JSON buffer, so the
|
||||||
|
// load is safe.
|
||||||
|
template <std::size_t MaxKeyLen>
|
||||||
|
simdjson_really_inline std::size_t scan_key_length(const char* p) noexcept {
|
||||||
|
// The SIMD paths scan only two 16-byte blocks (offsets 0..31), so a key
|
||||||
|
// whose closing quote sits at offset 32 would be missed. Cap at 31 to keep
|
||||||
|
// SIMD and scalar builds in agreement.
|
||||||
|
static_assert(MaxKeyLen <= 31, "MaxKeyLen must be <= 31 for current SIMD implementations");
|
||||||
|
#if SIMDJSON_KEY_SELECTOR_HAS_NEON
|
||||||
|
uint8x16_t v0 = vld1q_u8(reinterpret_cast<const uint8_t*>(p));
|
||||||
|
uint8x16_t cmp0 = vceqq_u8(v0, vdupq_n_u8('"'));
|
||||||
|
uint64_t m0 = vget_lane_u64(
|
||||||
|
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(cmp0), 4)), 0);
|
||||||
|
if constexpr (MaxKeyLen < 16) {
|
||||||
|
if (simdjson_likely(m0 != 0)) return std::size_t(__builtin_ctzll(m0)) >> 2;
|
||||||
|
return MaxKeyLen + 1;
|
||||||
|
} else {
|
||||||
|
uint8x16_t v1 = vld1q_u8(reinterpret_cast<const uint8_t*>(p) + 16);
|
||||||
|
uint8x16_t cmp1 = vceqq_u8(v1, vdupq_n_u8('"'));
|
||||||
|
uint64_t m1 = vget_lane_u64(
|
||||||
|
vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(cmp1), 4)), 0);
|
||||||
|
if (simdjson_likely(m0 != 0)) return std::size_t(__builtin_ctzll(m0)) >> 2;
|
||||||
|
if (m1 != 0) return 16 + (std::size_t(__builtin_ctzll(m1)) >> 2);
|
||||||
|
return MaxKeyLen + 1;
|
||||||
|
}
|
||||||
|
#elif SIMDJSON_KEY_SELECTOR_HAS_SSE2
|
||||||
|
__m128i v0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
|
||||||
|
__m128i cmp0 = _mm_cmpeq_epi8(v0, _mm_set1_epi8('"'));
|
||||||
|
unsigned m0 = static_cast<unsigned>(_mm_movemask_epi8(cmp0));
|
||||||
|
if constexpr (MaxKeyLen < 16) {
|
||||||
|
if (simdjson_likely(m0 != 0)) return std::size_t(__builtin_ctz(m0));
|
||||||
|
return MaxKeyLen + 1;
|
||||||
|
} else {
|
||||||
|
__m128i v1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + 16));
|
||||||
|
__m128i cmp1 = _mm_cmpeq_epi8(v1, _mm_set1_epi8('"'));
|
||||||
|
unsigned m1 = static_cast<unsigned>(_mm_movemask_epi8(cmp1));
|
||||||
|
if (simdjson_likely(m0 != 0)) return std::size_t(__builtin_ctz(m0));
|
||||||
|
if (m1 != 0) return 16 + std::size_t(__builtin_ctz(m1));
|
||||||
|
return MaxKeyLen + 1;
|
||||||
|
}
|
||||||
|
#else
|
||||||
|
for (std::size_t i = 0; i <= MaxKeyLen; ++i)
|
||||||
|
if (p[i] == '"') return i;
|
||||||
|
return MaxKeyLen + 1;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
// Byte-equal of p[0..len) against stored[0..len). stored is zero-padded past
|
||||||
|
// `len`. Input is read over 16 or 32 bytes (padded JSON buffer guaranteed).
|
||||||
|
template <std::size_t MaxKeyLen>
|
||||||
|
simdjson_really_inline bool compare_key_bytes(
|
||||||
|
const char* p, const char* stored, std::size_t len) noexcept {
|
||||||
|
alignas(16) static constexpr uint8_t idx16[16] =
|
||||||
|
{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
|
||||||
|
if constexpr (MaxKeyLen <= 16) {
|
||||||
|
#if SIMDJSON_KEY_SELECTOR_HAS_NEON
|
||||||
|
uint8x16_t vp = vld1q_u8(reinterpret_cast<const uint8_t*>(p));
|
||||||
|
uint8x16_t vs = vld1q_u8(reinterpret_cast<const uint8_t*>(stored));
|
||||||
|
uint8x16_t mask = vcltq_u8(vld1q_u8(idx16), vdupq_n_u8(static_cast<uint8_t>(len)));
|
||||||
|
uint8x16_t diff = veorq_u8(vandq_u8(vp, mask), vs);
|
||||||
|
// A 32-bit-lane horizontal max is enough to decide "all bytes equal"
|
||||||
|
// (diff is zero iff every 32-bit word is zero) and is cheaper than a
|
||||||
|
// byte-wide reduction.
|
||||||
|
return vmaxvq_u32(vreinterpretq_u32_u8(diff)) == 0;
|
||||||
|
#elif SIMDJSON_KEY_SELECTOR_HAS_SSE2
|
||||||
|
__m128i vp = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
|
||||||
|
__m128i vs = _mm_loadu_si128(reinterpret_cast<const __m128i*>(stored));
|
||||||
|
__m128i idx = _mm_load_si128(reinterpret_cast<const __m128i*>(idx16));
|
||||||
|
__m128i mask = _mm_cmplt_epi8(idx, _mm_set1_epi8(static_cast<char>(len)));
|
||||||
|
__m128i eq = _mm_cmpeq_epi8(_mm_and_si128(vp, mask), vs);
|
||||||
|
return _mm_movemask_epi8(eq) == 0xFFFF;
|
||||||
|
#else
|
||||||
|
for (std::size_t i = 0; i < len; ++i)
|
||||||
|
if (p[i] != stored[i]) return false;
|
||||||
|
return true;
|
||||||
|
#endif
|
||||||
|
} else if constexpr (MaxKeyLen <= 32) {
|
||||||
|
alignas(16) static constexpr uint8_t idx32_hi[16] =
|
||||||
|
{16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31};
|
||||||
|
#if SIMDJSON_KEY_SELECTOR_HAS_NEON
|
||||||
|
uint8x16_t vp_lo = vld1q_u8(reinterpret_cast<const uint8_t*>(p));
|
||||||
|
uint8x16_t vp_hi = vld1q_u8(reinterpret_cast<const uint8_t*>(p) + 16);
|
||||||
|
uint8x16_t vs_lo = vld1q_u8(reinterpret_cast<const uint8_t*>(stored));
|
||||||
|
uint8x16_t vs_hi = vld1q_u8(reinterpret_cast<const uint8_t*>(stored) + 16);
|
||||||
|
uint8x16_t lenv = vdupq_n_u8(static_cast<uint8_t>(len));
|
||||||
|
uint8x16_t m_lo = vcltq_u8(vld1q_u8(idx16), lenv);
|
||||||
|
uint8x16_t m_hi = vcltq_u8(vld1q_u8(idx32_hi), lenv);
|
||||||
|
uint8x16_t d_lo = veorq_u8(vandq_u8(vp_lo, m_lo), vs_lo);
|
||||||
|
uint8x16_t d_hi = veorq_u8(vandq_u8(vp_hi, m_hi), vs_hi);
|
||||||
|
return vmaxvq_u32(vreinterpretq_u32_u8(vorrq_u8(d_lo, d_hi))) == 0;
|
||||||
|
#elif SIMDJSON_KEY_SELECTOR_HAS_SSE2
|
||||||
|
__m128i vp_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
|
||||||
|
__m128i vp_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p + 16));
|
||||||
|
__m128i vs_lo = _mm_loadu_si128(reinterpret_cast<const __m128i*>(stored));
|
||||||
|
__m128i vs_hi = _mm_loadu_si128(reinterpret_cast<const __m128i*>(stored + 16));
|
||||||
|
__m128i lenv = _mm_set1_epi8(static_cast<char>(len));
|
||||||
|
__m128i m_lo = _mm_cmplt_epi8(_mm_load_si128(reinterpret_cast<const __m128i*>(idx16)), lenv);
|
||||||
|
__m128i m_hi = _mm_cmplt_epi8(_mm_load_si128(reinterpret_cast<const __m128i*>(idx32_hi)), lenv);
|
||||||
|
__m128i eq_lo = _mm_cmpeq_epi8(_mm_and_si128(vp_lo, m_lo), vs_lo);
|
||||||
|
__m128i eq_hi = _mm_cmpeq_epi8(_mm_and_si128(vp_hi, m_hi), vs_hi);
|
||||||
|
return (_mm_movemask_epi8(eq_lo) & _mm_movemask_epi8(eq_hi)) == 0xFFFF;
|
||||||
|
#else
|
||||||
|
for (std::size_t i = 0; i < len; ++i)
|
||||||
|
if (p[i] != stored[i]) return false;
|
||||||
|
return true;
|
||||||
|
#endif
|
||||||
|
} else {
|
||||||
|
for (std::size_t i = 0; i < len; ++i)
|
||||||
|
if (p[i] != stored[i]) return false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace key_selector_detail
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stateless, compile-time key selector.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* using sel_t = key_selector<"id", "text", "user">;
|
||||||
|
* std::size_t i = sel_t::match_raw(raw_key); // returns sel_t::size() on miss
|
||||||
|
*
|
||||||
|
* The perfect hash is built at compile time (gperf-style, with a
|
||||||
|
* Hash-and-Displace fallback) and only flat tables survive to runtime. All
|
||||||
|
* tables are static constexpr, so the lookup fully inlines.
|
||||||
|
*
|
||||||
|
* Limitations:
|
||||||
|
* - Each key must be at most 31 characters long (and no longer than
|
||||||
|
* SIMDJSON_PADDING). Longer keys trigger a compile-time error.
|
||||||
|
* - The number of keys should be moderate. The hard limit is 255 keys;
|
||||||
|
* compilation time grows with the number of keys, so prefer a few dozen at
|
||||||
|
* most per selector.
|
||||||
|
* - Keys must be distinct, non-empty, and free of backslash, double-quote and
|
||||||
|
* null bytes (matching is done against the raw, unescaped JSON key bytes).
|
||||||
|
*/
|
||||||
|
template <constevalutil::fixed_string... Keys>
|
||||||
|
struct key_selector {
|
||||||
|
static constexpr std::size_t N = sizeof...(Keys);
|
||||||
|
static_assert(N > 0, "key_selector requires at least one key");
|
||||||
|
static_assert(N <= 255,"key_selector supports at most 255 keys");
|
||||||
|
|
||||||
|
static constexpr std::array<std::string_view, N> keys{ Keys.view()... };
|
||||||
|
static constexpr std::size_t max_key_len = key_selector_detail::compute_max_key_len<N>(keys);
|
||||||
|
static_assert(max_key_len <= SIMDJSON_PADDING,
|
||||||
|
"key longer than SIMDJSON_PADDING is not supported");
|
||||||
|
// The SIMD key-length scan covers offsets 0..31 only; a 32-character key's
|
||||||
|
// closing quote lands at offset 32 and would be silently missed on
|
||||||
|
// NEON/SSE2 while still matching in scalar builds. Cap at 31 so the result
|
||||||
|
// is identical across implementations.
|
||||||
|
static_assert(max_key_len <= 31,
|
||||||
|
"key_selector keys must be at most 31 characters long");
|
||||||
|
|
||||||
|
static constexpr auto result = key_selector_detail::compute_phf<N>(keys);
|
||||||
|
static constexpr std::size_t table_size = result.table_size;
|
||||||
|
|
||||||
|
static constexpr auto phf =
|
||||||
|
key_selector_detail::build_phf_data<N, table_size, max_key_len>(keys, result);
|
||||||
|
|
||||||
|
static constexpr std::size_t size() noexcept { return N; }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Look up a JSON key. rjs must point just after an opening quote in a padded
|
||||||
|
* simdjson buffer. Returns the selector index in [0, N) on match, or N on miss.
|
||||||
|
*/
|
||||||
|
static simdjson_really_inline std::size_t match_raw(raw_json_string rjs) noexcept {
|
||||||
|
const char* p = rjs.raw();
|
||||||
|
std::size_t len = key_selector_detail::scan_key_length<max_key_len>(p);
|
||||||
|
if (len == 0 || len > max_key_len) { return N; }
|
||||||
|
|
||||||
|
std::size_t slot;
|
||||||
|
if (phf.num_positions == key_selector_detail::HD_MODE) {
|
||||||
|
// Hash-and-Displace: bucket displacement + per-key hash.
|
||||||
|
std::string_view key(p, len);
|
||||||
|
std::size_t bucket = key_selector_detail::hd_bucket_hash(key);
|
||||||
|
std::size_t kh = (phf.hd_hash_variant == key_selector_detail::HD_HASH_2BYTE_FLAG)
|
||||||
|
? key_selector_detail::hd_key_hash_2(key)
|
||||||
|
: key_selector_detail::hd_key_hash_4(key);
|
||||||
|
slot = (phf.asso_values[0][bucket] + kh) & (table_size - 1);
|
||||||
|
} else {
|
||||||
|
// gperf: h = len + sum of asso_values over the selected positions.
|
||||||
|
// positions / num_positions / asso_values are compile-time constants,
|
||||||
|
// so this loop fully unrolls. The idx < len guard mirrors the
|
||||||
|
// generator's char_at()-> 256 -> skip behavior for out-of-range
|
||||||
|
// positions (required: arbitrary positions may exceed a key's length).
|
||||||
|
std::size_t h = len;
|
||||||
|
for (std::uint8_t i = 0; i < phf.num_positions; ++i) {
|
||||||
|
std::uint8_t pos = phf.positions[i];
|
||||||
|
std::size_t idx = (pos == key_selector_detail::POS_LAST_CHAR)
|
||||||
|
? (len - std::size_t{1})
|
||||||
|
: static_cast<std::size_t>(pos);
|
||||||
|
if (idx < len) {
|
||||||
|
h += phf.asso_values[i][static_cast<unsigned char>(p[idx])];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
slot = h & (table_size - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::uint8_t ki = phf.slot_to_key[slot];
|
||||||
|
if (ki >= N) { return N; }
|
||||||
|
if (phf.slot_key_len[slot] != len) { return N; }
|
||||||
|
if (!key_selector_detail::compare_key_bytes<max_key_len>(
|
||||||
|
p, phf.slot_key_bytes[slot].data(), len)) { return N; }
|
||||||
|
return ki;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Return the key text at selector index i (i in [0, N)). */
|
||||||
|
static constexpr std::string_view key_at(std::size_t i) noexcept {
|
||||||
|
return keys[i];
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace ondemand
|
||||||
|
} // namespace SIMDJSON_IMPLEMENTATION
|
||||||
|
} // namespace simdjson
|
||||||
|
|
||||||
|
#endif // SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
|
||||||
|
#endif // SIMDJSON_GENERIC_ONDEMAND_KEY_SELECTOR_H
|
||||||
@@ -63,6 +63,49 @@ simdjson_inline simdjson_result<value> object::find_field(const std::string_view
|
|||||||
return value(iter.child());
|
return value(iter.child());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
template <typename Selector, typename Func>
|
||||||
|
simdjson_flatten simdjson_inline for_each_result object::for_each(Func&& on_match) noexcept {
|
||||||
|
// Single pass driven directly by the value_iterator, mirroring
|
||||||
|
// find_field_unordered_raw + value(iter.child()). Compared to walking via
|
||||||
|
// object_iterator/field, this avoids constructing a simdjson_result<field> and
|
||||||
|
// a field (key + value) for every field -- and the development-check bookkeeping
|
||||||
|
// in object_iterator -- building a value only for the fields that actually match.
|
||||||
|
// We operate on a copy of the iterator, as object::begin() would.
|
||||||
|
value_iterator it = iter;
|
||||||
|
std::array<bool, Selector::size()> seen{};
|
||||||
|
std::size_t matched = 0;
|
||||||
|
while (it.is_open()) {
|
||||||
|
raw_json_string key;
|
||||||
|
error_code error = it.field_key().get(key);
|
||||||
|
if (error) { it.abandon(); return {error, matched}; }
|
||||||
|
// Advance past the ':' and descend onto the value.
|
||||||
|
if ((error = it.field_value())) { it.abandon(); return {error, matched}; }
|
||||||
|
std::size_t idx = Selector::match_raw(key);
|
||||||
|
if (idx < Selector::size() && !seen[idx]) {
|
||||||
|
seen[idx] = true;
|
||||||
|
value matched_value(it.child());
|
||||||
|
// The callback may return either void or an error_code. When it returns an
|
||||||
|
// error_code, we stop at the first non-SUCCESS result and propagate it so
|
||||||
|
// the caller can surface value-parse errors (for example, a type mismatch
|
||||||
|
// on a matched field). A void-returning callback is responsible for
|
||||||
|
// handling its own errors.
|
||||||
|
if constexpr (std::is_same_v<decltype(on_match(idx, matched_value)), error_code>) {
|
||||||
|
if ((error = on_match(idx, matched_value))) { return {error, matched}; }
|
||||||
|
} else {
|
||||||
|
on_match(idx, matched_value);
|
||||||
|
}
|
||||||
|
if (++matched >= Selector::size()) { break; }
|
||||||
|
}
|
||||||
|
// Skip the value (a no-op if the callback consumed it) and step to the next
|
||||||
|
// field; has_next_field() ends the container on '}', which closes the loop.
|
||||||
|
if ((error = it.skip_child())) { it.abandon(); return {error, matched}; }
|
||||||
|
if ((error = it.has_next_field().error())) { return {error, matched}; }
|
||||||
|
}
|
||||||
|
return {SUCCESS, matched};
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
simdjson_inline simdjson_result<object> object::start(value_iterator &iter) noexcept {
|
simdjson_inline simdjson_result<object> object::start(value_iterator &iter) noexcept {
|
||||||
SIMDJSON_TRY( iter.start_object().error() );
|
SIMDJSON_TRY( iter.start_object().error() );
|
||||||
return object(iter);
|
return object(iter);
|
||||||
@@ -326,6 +369,7 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjs
|
|||||||
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first).find_field(key);
|
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first).find_field(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::at_pointer(std::string_view json_pointer) noexcept {
|
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::at_pointer(std::string_view json_pointer) noexcept {
|
||||||
if (error()) { return error(); }
|
if (error()) { return error(); }
|
||||||
return first.at_pointer(json_pointer);
|
return first.at_pointer(json_pointer);
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
#include "simdjson/generic/ondemand/base.h"
|
#include "simdjson/generic/ondemand/base.h"
|
||||||
#include "simdjson/generic/implementation_simdjson_result_base.h"
|
#include "simdjson/generic/implementation_simdjson_result_base.h"
|
||||||
#include "simdjson/generic/ondemand/value_iterator.h"
|
#include "simdjson/generic/ondemand/value_iterator.h"
|
||||||
|
#include "simdjson/generic/ondemand/key_selector.h"
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#if SIMDJSON_STATIC_REFLECTION && SIMDJSON_SUPPORTS_CONCEPTS
|
#if SIMDJSON_STATIC_REFLECTION && SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
#include "simdjson/generic/ondemand/json_string_builder.h" // for constevalutil::fixed_string
|
#include "simdjson/generic/ondemand/json_string_builder.h" // for constevalutil::fixed_string
|
||||||
@@ -15,6 +16,22 @@ namespace simdjson {
|
|||||||
namespace SIMDJSON_IMPLEMENTATION {
|
namespace SIMDJSON_IMPLEMENTATION {
|
||||||
namespace ondemand {
|
namespace ondemand {
|
||||||
|
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
/**
|
||||||
|
* Result of object::for_each: the first error encountered (SUCCESS if none) and
|
||||||
|
* the number of distinct selector keys that matched during the walk. A
|
||||||
|
* matched_count equal to Selector::size() means every selected key was present
|
||||||
|
* in the object. Implicitly converts to error_code so existing callers that only
|
||||||
|
* care about the error (including SIMDJSON_TRY and the test ASSERT_* macros) keep
|
||||||
|
* working unchanged.
|
||||||
|
*/
|
||||||
|
struct for_each_result {
|
||||||
|
error_code error{SUCCESS};
|
||||||
|
std::size_t matched_count{0};
|
||||||
|
constexpr operator error_code() const noexcept { return error; }
|
||||||
|
};
|
||||||
|
#endif
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A forward-only JSON object field iterator.
|
* A forward-only JSON object field iterator.
|
||||||
*/
|
*/
|
||||||
@@ -122,6 +139,41 @@ public:
|
|||||||
/** @overload simdjson_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
|
/** @overload simdjson_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
|
||||||
simdjson_inline simdjson_result<value> operator[](std::string_view key) && noexcept;
|
simdjson_inline simdjson_result<value> operator[](std::string_view key) && noexcept;
|
||||||
|
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
/**
|
||||||
|
* Walk this object once and invoke on_match(selector_index, value) for each
|
||||||
|
* field whose key is in the compile-time key_selector Selector, in JSON order
|
||||||
|
* (first occurrence of a duplicate key wins). Iteration stops once all
|
||||||
|
* Selector::size() keys have matched or the object ends. The value is consumed
|
||||||
|
* in place, so this is a low-overhead way to extract a known set of fields
|
||||||
|
* regardless of their order in the JSON.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* using sel_t = ondemand::key_selector<"id", "text", "user">;
|
||||||
|
* obj.for_each<sel_t>([&](std::size_t i, ondemand::value v) {
|
||||||
|
* switch (i) { case 0: ...; case 1: ...; }
|
||||||
|
* });
|
||||||
|
*
|
||||||
|
* Limitations (see key_selector): each key must be at most 31 characters long,
|
||||||
|
* and the number of keys should be moderate (hard limit 255; a handful is
|
||||||
|
* best, as the compile-time perfect hash may fail or slow compilation for
|
||||||
|
* large key sets). They keys must be distinct, non-empty, and free of backslash, double-quote and
|
||||||
|
* null bytes.
|
||||||
|
*
|
||||||
|
* The callback may return either void or an error_code. When it returns an
|
||||||
|
* error_code, the walk stops at the first non-SUCCESS result and that error is
|
||||||
|
* returned, which lets the callback surface value-parse errors.
|
||||||
|
*
|
||||||
|
* @returns a for_each_result holding the first error encountered while walking
|
||||||
|
* the object (including any error returned by the callback, SUCCESS if
|
||||||
|
* none) and the number of distinct selector keys that matched. The
|
||||||
|
* result converts implicitly to error_code, so callers that only need
|
||||||
|
* the error can ignore the count.
|
||||||
|
*/
|
||||||
|
template <typename Selector, typename Func>
|
||||||
|
simdjson_inline for_each_result for_each(Func&& on_match) noexcept;
|
||||||
|
#endif
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get the value associated with the given JSON pointer. We use the RFC 6901
|
* Get the value associated with the given JSON pointer. We use the RFC 6901
|
||||||
* https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
|
* https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
#include <limits>
|
#include <limits>
|
||||||
#if SIMDJSON_STATIC_REFLECTION
|
#if SIMDJSON_STATIC_REFLECTION
|
||||||
#include <meta>
|
#include <meta>
|
||||||
|
#include <vector>
|
||||||
// #include <static_reflection> // for std::define_static_string - header not available yet
|
// #include <static_reflection> // for std::define_static_string - header not available yet
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@@ -268,6 +269,135 @@ constexpr bool user_defined_type = (std::is_class_v<T>
|
|||||||
!concepts::appendable_containers<T>);
|
!concepts::appendable_containers<T>);
|
||||||
|
|
||||||
|
|
||||||
|
#if defined(SIMDJSON_USE_KEY_SELECTOR_REFLECTION) && SIMDJSON_USE_KEY_SELECTOR_REFLECTION
|
||||||
|
|
||||||
|
// Experimental: deserialize a reflected struct using a compile-time key_selector
|
||||||
|
// and a single object::for_each pass (perfect-hash key matching) instead of one
|
||||||
|
// obj[key] lookup per member. Enable with -DSIMDJSON_USE_KEY_SELECTOR_REFLECTION=1.
|
||||||
|
namespace key_selector_reflection_detail {
|
||||||
|
|
||||||
|
// A member participates if it is public, non-const, and not annotated to skip.
|
||||||
|
consteval bool is_eligible_member(std::meta::info mem) {
|
||||||
|
return !std::meta::is_const(mem) && std::meta::is_public(mem)
|
||||||
|
&& std::meta::annotations_of_with_type(mem, ^^simdjson::detail::skip_tag).empty();
|
||||||
|
}
|
||||||
|
|
||||||
|
// JSON key name for `mem`, as a constevalutil::fixed_string usable as an NTTP.
|
||||||
|
template <auto mem>
|
||||||
|
consteval auto member_key_fixed_string() {
|
||||||
|
constexpr std::string_view key{ simdjson::get_json_key_name<mem>() };
|
||||||
|
char buffer[key.size() + 1] = {};
|
||||||
|
for (std::size_t i = 0; i < key.size(); ++i) { buffer[i] = key[i]; }
|
||||||
|
return constevalutil::fixed_string<key.size() + 1>(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
// key_selector template arguments (one fixed_string per eligible member), in
|
||||||
|
// declaration order.
|
||||||
|
template <typename T>
|
||||||
|
consteval std::vector<std::meta::info> selector_key_args() {
|
||||||
|
std::vector<std::meta::info> args;
|
||||||
|
template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
|
||||||
|
if constexpr (is_eligible_member(mem)) {
|
||||||
|
args.push_back(std::meta::reflect_constant(member_key_fixed_string<mem>()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return args;
|
||||||
|
}
|
||||||
|
|
||||||
|
// key_selector whose keys are exactly T's eligible members (index i <-> i-th).
|
||||||
|
template <typename T>
|
||||||
|
using selector_for = typename [: std::meta::substitute(
|
||||||
|
^^SIMDJSON_IMPLEMENTATION::ondemand::key_selector, selector_key_args<T>()) :];
|
||||||
|
|
||||||
|
// True when none of T's eligible members is an optional type, i.e. every member
|
||||||
|
// is required. In that case presence can be checked with a single match count
|
||||||
|
// instead of a per-member "seen" array.
|
||||||
|
template <typename T>
|
||||||
|
consteval bool all_eligible_members_required() {
|
||||||
|
bool all_required = true;
|
||||||
|
template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
|
||||||
|
if constexpr (is_eligible_member(mem)) {
|
||||||
|
if constexpr (concepts::optional_type<typename [: std::meta::type_of(mem) :]>) {
|
||||||
|
all_required = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return all_required;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace key_selector_reflection_detail
|
||||||
|
|
||||||
|
template <typename T, typename ValT>
|
||||||
|
requires(user_defined_type<T> && std::is_class_v<T>)
|
||||||
|
error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
|
||||||
|
SIMDJSON_IMPLEMENTATION::ondemand::object obj;
|
||||||
|
if constexpr (std::is_same_v<std::remove_cvref_t<ValT>, SIMDJSON_IMPLEMENTATION::ondemand::object>) {
|
||||||
|
obj = val;
|
||||||
|
} else {
|
||||||
|
SIMDJSON_TRY(val.get_object().get(obj));
|
||||||
|
}
|
||||||
|
using selector = key_selector_reflection_detail::selector_for<T>;
|
||||||
|
if constexpr (key_selector_reflection_detail::all_eligible_members_required<T>()) {
|
||||||
|
// Fast path: every member is required. A single for_each pass parses each
|
||||||
|
// matched field; the returned match count then tells us whether every member
|
||||||
|
// was present (matched_count == selector::size()) without a per-member "seen"
|
||||||
|
// array. A value-parse error (e.g. a type mismatch) is propagated by for_each.
|
||||||
|
auto walk = obj.template for_each<selector>(
|
||||||
|
[&](std::size_t matched_index, SIMDJSON_IMPLEMENTATION::ondemand::value field_value) -> error_code {
|
||||||
|
std::size_t counter = 0;
|
||||||
|
template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
|
||||||
|
if constexpr (key_selector_reflection_detail::is_eligible_member(mem)) {
|
||||||
|
if (matched_index == counter) { return field_value.get(out.[:mem:]); }
|
||||||
|
++counter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return SUCCESS;
|
||||||
|
});
|
||||||
|
if (walk.error) { return walk.error; }
|
||||||
|
// A missing required member shows up as a short match count and is reported as
|
||||||
|
// NO_SUCH_FIELD, mirroring the ordered obj[key] path.
|
||||||
|
if (walk.matched_count != selector::size()) { return NO_SUCH_FIELD; }
|
||||||
|
return SUCCESS;
|
||||||
|
} else {
|
||||||
|
std::array<bool, selector::size()> seen_member{};
|
||||||
|
// Single pass over the object: each field whose key matches a member yields its
|
||||||
|
// selector index, which we map back to the corresponding member. The callback
|
||||||
|
// returns an error_code so that a value-parse error (e.g. a type mismatch on a
|
||||||
|
// matched field) is propagated by for_each instead of being silently dropped.
|
||||||
|
error_code walk_error = obj.template for_each<selector>(
|
||||||
|
[&](std::size_t matched_index, SIMDJSON_IMPLEMENTATION::ondemand::value field_value) -> error_code {
|
||||||
|
std::size_t counter = 0;
|
||||||
|
error_code field_error = SUCCESS;
|
||||||
|
template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
|
||||||
|
if constexpr (key_selector_reflection_detail::is_eligible_member(mem)) {
|
||||||
|
if (matched_index == counter) {
|
||||||
|
seen_member[counter] = true;
|
||||||
|
field_error = field_value.get(out.[:mem:]);
|
||||||
|
}
|
||||||
|
++counter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return field_error;
|
||||||
|
});
|
||||||
|
if (walk_error) { return walk_error; }
|
||||||
|
// Required (non-optional) members must be present: a missing one is reported as
|
||||||
|
// NO_SUCH_FIELD, mirroring the ordered obj[key] path. Optional members may be
|
||||||
|
// absent.
|
||||||
|
std::size_t check_counter = 0;
|
||||||
|
template for (constexpr auto mem : std::define_static_array(std::meta::nonstatic_data_members_of(^^T, std::meta::access_context::unchecked()))) {
|
||||||
|
if constexpr (key_selector_reflection_detail::is_eligible_member(mem)) {
|
||||||
|
if constexpr (!concepts::optional_type<decltype(out.[:mem:])>) {
|
||||||
|
if (!seen_member[check_counter]) { return NO_SUCH_FIELD; }
|
||||||
|
}
|
||||||
|
++check_counter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return SUCCESS;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
template <typename T, typename ValT>
|
template <typename T, typename ValT>
|
||||||
requires(user_defined_type<T> && std::is_class_v<T>)
|
requires(user_defined_type<T> && std::is_class_v<T>)
|
||||||
error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
|
error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
|
||||||
@@ -300,6 +430,8 @@ error_code tag_invoke(deserialize_tag, ValT &val, T &out) noexcept {
|
|||||||
return simdjson::SUCCESS;
|
return simdjson::SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#endif // SIMDJSON_USE_KEY_SELECTOR_REFLECTION
|
||||||
|
|
||||||
// Support for enum deserialization - deserialize from string representation using expand approach from P2996R12
|
// Support for enum deserialization - deserialize from string representation using expand approach from P2996R12
|
||||||
template <typename T, typename ValT>
|
template <typename T, typename ValT>
|
||||||
requires(std::is_enum_v<T>)
|
requires(std::is_enum_v<T>)
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ add_cpp_test(ondemand_nan_inf_tests LABELS ondemand acceptance
|
|||||||
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_number_in_string_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_number_in_string_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
|
||||||
|
add_cpp_test(ondemand_object_find_field_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
|
||||||
@@ -37,6 +38,10 @@ add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance
|
|||||||
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_custom_types_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_custom_types_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_custom_types_document_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_custom_types_document_tests LABELS ondemand acceptance per_implementation)
|
||||||
|
# Reflective deserialization through the experimental key-selector path. The
|
||||||
|
# SIMDJSON_USE_KEY_SELECTOR_REFLECTION macro is defined at the top of the test
|
||||||
|
# source itself, so this is an ordinary test target.
|
||||||
|
add_cpp_test(ondemand_key_selector_reflection_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_stl_types_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_stl_types_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_convert_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_convert_tests LABELS ondemand acceptance per_implementation)
|
||||||
add_cpp_test(ondemand_unknown_tests LABELS ondemand acceptance per_implementation)
|
add_cpp_test(ondemand_unknown_tests LABELS ondemand acceptance per_implementation)
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
// Exercises the experimental key-selector-based reflective deserializer.
|
||||||
|
//
|
||||||
|
// We enable the option here (before including simdjson) so the reflective
|
||||||
|
// tag_invoke uses a compile-time key_selector + object::for_each single pass
|
||||||
|
// instead of one obj[key] lookup per member. The point of the option is to
|
||||||
|
// deserialize structs correctly even when the JSON keys are NOT in declaration
|
||||||
|
// order, so the tests below deliberately scramble the key order.
|
||||||
|
#define SIMDJSON_USE_KEY_SELECTOR_REFLECTION 1
|
||||||
|
#include "simdjson.h"
|
||||||
|
#include <optional>
|
||||||
|
#include <string>
|
||||||
|
#include "test_ondemand.h"
|
||||||
|
|
||||||
|
#if SIMDJSON_STATIC_REFLECTION
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using namespace simdjson;
|
||||||
|
|
||||||
|
// Default member initializers keep -Weffc++ happy; reflective deserialization
|
||||||
|
// overwrites these fields from the JSON.
|
||||||
|
struct User {
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string screen_name{};
|
||||||
|
std::optional<std::string> location{};
|
||||||
|
bool verified = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Tweet {
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string text{};
|
||||||
|
uint64_t retweet_count = 0;
|
||||||
|
User user{};
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Flat {
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string text{};
|
||||||
|
uint64_t retweet_count = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
bool flat_out_of_order() {
|
||||||
|
TEST_START();
|
||||||
|
// Keys deliberately out of declaration order.
|
||||||
|
auto json = R"({ "retweet_count": 7, "id": 12345, "text": "hello world" })"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc;
|
||||||
|
ASSERT_SUCCESS(parser.iterate(json).get(doc));
|
||||||
|
Flat f;
|
||||||
|
ASSERT_SUCCESS(doc.get(f));
|
||||||
|
ASSERT_EQUAL(f.id, 12345);
|
||||||
|
ASSERT_EQUAL(f.text, "hello world");
|
||||||
|
ASSERT_EQUAL(f.retweet_count, 7);
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool nested_out_of_order() {
|
||||||
|
TEST_START();
|
||||||
|
// Both the outer and the nested object have scrambled key order, and the
|
||||||
|
// optional "location" is present here.
|
||||||
|
auto json = R"({
|
||||||
|
"user": {
|
||||||
|
"verified": true,
|
||||||
|
"location": "Montreal",
|
||||||
|
"screen_name": "ayuu0123",
|
||||||
|
"id": 1186275104
|
||||||
|
},
|
||||||
|
"retweet_count": 3,
|
||||||
|
"text": "nested",
|
||||||
|
"id": 99
|
||||||
|
})"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc;
|
||||||
|
ASSERT_SUCCESS(parser.iterate(json).get(doc));
|
||||||
|
Tweet t;
|
||||||
|
ASSERT_SUCCESS(doc.get(t));
|
||||||
|
ASSERT_EQUAL(t.id, 99);
|
||||||
|
ASSERT_EQUAL(t.text, "nested");
|
||||||
|
ASSERT_EQUAL(t.retweet_count, 3);
|
||||||
|
ASSERT_EQUAL(t.user.id, 1186275104);
|
||||||
|
ASSERT_EQUAL(t.user.screen_name, "ayuu0123");
|
||||||
|
ASSERT_TRUE(t.user.location.has_value());
|
||||||
|
ASSERT_EQUAL(t.user.location.value(), "Montreal");
|
||||||
|
ASSERT_TRUE(t.user.verified);
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool optional_missing() {
|
||||||
|
TEST_START();
|
||||||
|
// "location" is absent: the optional member must be left empty, not error.
|
||||||
|
auto json = R"({
|
||||||
|
"user": { "id": 7, "screen_name": "nobody", "verified": false },
|
||||||
|
"id": 1, "text": "t", "retweet_count": 0
|
||||||
|
})"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc;
|
||||||
|
ASSERT_SUCCESS(parser.iterate(json).get(doc));
|
||||||
|
Tweet t;
|
||||||
|
ASSERT_SUCCESS(doc.get(t));
|
||||||
|
ASSERT_EQUAL(t.user.id, 7);
|
||||||
|
ASSERT_EQUAL(t.user.screen_name, "nobody");
|
||||||
|
ASSERT_FALSE(t.user.location.has_value());
|
||||||
|
ASSERT_FALSE(t.user.verified);
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
|
||||||
|
bool run() {
|
||||||
|
return flat_out_of_order() &&
|
||||||
|
nested_out_of_order() &&
|
||||||
|
optional_missing() &&
|
||||||
|
true;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
int main(int argc, char *argv[]) {
|
||||||
|
return test_main(argc, argv, run);
|
||||||
|
}
|
||||||
|
|
||||||
|
#else
|
||||||
|
|
||||||
|
int main() { return 0; }
|
||||||
|
|
||||||
|
#endif
|
||||||
@@ -173,6 +173,218 @@ namespace object_tests {
|
|||||||
TEST_SUCCEED();
|
TEST_SUCCEED();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
bool object_find_field_key_selector() {
|
||||||
|
TEST_START();
|
||||||
|
auto json = R"({ "name": "John", "age": 30, "city": "New York" })"_padded;
|
||||||
|
using sel_t = ondemand::key_selector<"name", "age", "city">;
|
||||||
|
|
||||||
|
// The selector maps keys to compile-time indices.
|
||||||
|
ASSERT_EQUAL(sel_t::size(), 3);
|
||||||
|
ASSERT_EQUAL(sel_t::key_at(0), "name");
|
||||||
|
ASSERT_EQUAL(sel_t::key_at(1), "age");
|
||||||
|
ASSERT_EQUAL(sel_t::key_at(2), "city");
|
||||||
|
|
||||||
|
SUBTEST("ondemand::object with key_selector for_each", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
|
ondemand::object object;
|
||||||
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
|
|
||||||
|
std::array<bool, 3> seen{};
|
||||||
|
std::string_view name_val{};
|
||||||
|
uint64_t age_val{};
|
||||||
|
std::string_view city_val{};
|
||||||
|
// for_each only yields indices in [0, size()), so index is always < 3 here.
|
||||||
|
ASSERT_SUCCESS( object.for_each<sel_t>([&](std::size_t index, ondemand::value v) {
|
||||||
|
seen[index] = true;
|
||||||
|
switch (index) {
|
||||||
|
case 0: { std::string_view s; if (!v.get(s)) { name_val = s; } break; }
|
||||||
|
case 1: { uint64_t n; if (!v.get(n)) { age_val = n; } break; }
|
||||||
|
case 2: { std::string_view s; if (!v.get(s)) { city_val = s; } break; }
|
||||||
|
}
|
||||||
|
}) );
|
||||||
|
|
||||||
|
ASSERT_TRUE(seen[0] && seen[1] && seen[2]);
|
||||||
|
ASSERT_EQUAL(name_val, "John");
|
||||||
|
ASSERT_EQUAL(age_val, 30);
|
||||||
|
ASSERT_EQUAL(city_val, "New York");
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}));
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Regression test for the for_each callback error contract: when the callback
|
||||||
|
// returns an error_code, for_each stops at the first non-SUCCESS result and
|
||||||
|
// returns it (so a value-parse error is no longer silently dropped).
|
||||||
|
bool object_for_each_callback_error() {
|
||||||
|
TEST_START();
|
||||||
|
using sel_t = ondemand::key_selector<"id", "name">;
|
||||||
|
auto json = R"({ "id": 7, "name": "Daniel" })"_padded;
|
||||||
|
|
||||||
|
// Happy path: an error_code-returning callback extracts the values and
|
||||||
|
// for_each returns SUCCESS.
|
||||||
|
SUBTEST("error_code callback success", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
|
ondemand::object object;
|
||||||
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
|
uint64_t id{};
|
||||||
|
std::string_view name{};
|
||||||
|
ASSERT_SUCCESS( object.for_each<sel_t>([&](std::size_t index, ondemand::value v) -> simdjson::error_code {
|
||||||
|
switch (index) {
|
||||||
|
case 0: { return v.get(id); }
|
||||||
|
case 1: { return v.get(name); }
|
||||||
|
default: { return SUCCESS; }
|
||||||
|
}
|
||||||
|
}) );
|
||||||
|
ASSERT_EQUAL(id, 7);
|
||||||
|
ASSERT_EQUAL(name, "Daniel");
|
||||||
|
return true;
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Error path: a value-parse error inside the callback (here, reading the
|
||||||
|
// integer "id" as a string) is propagated by for_each.
|
||||||
|
SUBTEST("error_code callback propagates error", test_ondemand_doc(json, [&](auto doc_result) {
|
||||||
|
ondemand::object object;
|
||||||
|
ASSERT_SUCCESS( doc_result.get(object) );
|
||||||
|
ASSERT_ERROR( object.for_each<sel_t>([&](std::size_t index, ondemand::value v) -> simdjson::error_code {
|
||||||
|
std::string_view s;
|
||||||
|
if (index == 0) { return v.get(s); } // wrong type on purpose
|
||||||
|
return SUCCESS;
|
||||||
|
}), INCORRECT_TYPE );
|
||||||
|
return true;
|
||||||
|
}));
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
// Mirrors doc/basics.md "Key selectors", Example 1 (top-level fields).
|
||||||
|
bool key_selector_example_toplevel() {
|
||||||
|
TEST_START();
|
||||||
|
auto json = R"({ "name": "Daniel", "age": 42, "city": "Montreal" })"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
ondemand::object obj = doc.get_object();
|
||||||
|
|
||||||
|
using fields = ondemand::key_selector<"name", "city">;
|
||||||
|
|
||||||
|
std::string_view name, city;
|
||||||
|
obj.for_each<fields>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: name = std::string_view(v); break; // "name"
|
||||||
|
case 1: city = std::string_view(v); break; // "city"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
ASSERT_EQUAL(name, "Daniel");
|
||||||
|
ASSERT_EQUAL(city, "Montreal");
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mirrors doc/basics.md "Key selectors", Example 2 (nested object).
|
||||||
|
bool key_selector_example_nested() {
|
||||||
|
TEST_START();
|
||||||
|
auto json = R"({ "user": { "id": 1186275104, "screen_name": "ayuu0123", "verified": false } })"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
ondemand::object user = doc.find_field("user").get_object();
|
||||||
|
|
||||||
|
using user_fields = ondemand::key_selector<"id", "screen_name">;
|
||||||
|
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string_view handle;
|
||||||
|
user.for_each<user_fields>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: id = uint64_t(v); break; // "id"
|
||||||
|
case 1: handle = std::string_view(v); break; // "screen_name"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
ASSERT_EQUAL(id, 1186275104);
|
||||||
|
ASSERT_EQUAL(handle, "ayuu0123");
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mirrors doc/basics.md "Key selectors", Example 3 (a selected value that is
|
||||||
|
// itself an object, processed with a nested for_each).
|
||||||
|
bool key_selector_example_inner_object() {
|
||||||
|
TEST_START();
|
||||||
|
auto json = R"({
|
||||||
|
"id": 42,
|
||||||
|
"author": { "name": "Daniel", "handle": "lemire" },
|
||||||
|
"title": "On Demand"
|
||||||
|
})"_padded;
|
||||||
|
ondemand::parser parser;
|
||||||
|
ondemand::document doc = parser.iterate(json);
|
||||||
|
ondemand::object obj = doc.get_object();
|
||||||
|
|
||||||
|
using post_fields = ondemand::key_selector<"id", "author", "title">;
|
||||||
|
using author_fields = ondemand::key_selector<"name", "handle">;
|
||||||
|
|
||||||
|
uint64_t id = 0;
|
||||||
|
std::string_view title, author_name, author_handle;
|
||||||
|
obj.for_each<post_fields>([&](std::size_t i, ondemand::value v) {
|
||||||
|
switch (i) {
|
||||||
|
case 0: id = uint64_t(v); break; // "id"
|
||||||
|
case 1: { // "author" is itself an object
|
||||||
|
ondemand::object author = v.get_object();
|
||||||
|
author.for_each<author_fields>([&](std::size_t j, ondemand::value av) {
|
||||||
|
switch (j) {
|
||||||
|
case 0: author_name = std::string_view(av); break; // "name"
|
||||||
|
case 1: author_handle = std::string_view(av); break; // "handle"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
case 2: title = std::string_view(v); break; // "title"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
ASSERT_EQUAL(id, 42);
|
||||||
|
ASSERT_EQUAL(title, "On Demand");
|
||||||
|
ASSERT_EQUAL(author_name, "Daniel");
|
||||||
|
ASSERT_EQUAL(author_handle, "lemire");
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
// The key_selector's matcher (match_raw, the perfect hash) must return the
|
||||||
|
// right selector index for every key, including tricky cases: keys that are
|
||||||
|
// prefixes of one another, keys that extend a real key, a long key, and misses.
|
||||||
|
// This guards the prefix/length disambiguation against the hash.
|
||||||
|
bool key_selector_matchers_agree() {
|
||||||
|
TEST_START();
|
||||||
|
// Probe builds "<key>\"" in a padded buffer and checks match_raw returns the
|
||||||
|
// expected selector index (or N for a miss).
|
||||||
|
auto probe = [](auto sel_tag, std::string_view key, std::size_t expected) -> bool {
|
||||||
|
using sel = decltype(sel_tag);
|
||||||
|
char buf[64] = {};
|
||||||
|
for (size_t i = 0; i < key.size(); ++i) { buf[i] = key[i]; }
|
||||||
|
buf[key.size()] = '"';
|
||||||
|
ondemand::raw_json_string r(reinterpret_cast<const uint8_t*>(buf));
|
||||||
|
ASSERT_EQUAL(sel::match_raw(r), expected);
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
// Selector with prefix keys and a 30-character key.
|
||||||
|
using small_sel = ondemand::key_selector<"a", "ab", "abc", "id", "name",
|
||||||
|
"abcdefghijklmnopqrstuvwxyz1234">;
|
||||||
|
std::size_t i = 0;
|
||||||
|
for (auto k : {"a", "ab", "abc", "id", "name", "abcdefghijklmnopqrstuvwxyz1234"}) {
|
||||||
|
if (!probe(small_sel{}, k, i++)) { return false; }
|
||||||
|
}
|
||||||
|
for (auto k : {"x", "abcd", "nam", "names", "i", "ids", "zzzzz", ""}) {
|
||||||
|
if (!probe(small_sel{}, k, small_sel::size())) { return false; }
|
||||||
|
}
|
||||||
|
// Larger selector exercising the same matcher.
|
||||||
|
using big_sel = ondemand::key_selector<"k00","k01","k02","k03","k04","k05",
|
||||||
|
"k06","k07","k08","k09","k10","k11">;
|
||||||
|
if (!probe(big_sel{}, "k00", 0)) { return false; }
|
||||||
|
if (!probe(big_sel{}, "k05", 5)) { return false; }
|
||||||
|
if (!probe(big_sel{}, "k11", 11)) { return false; }
|
||||||
|
for (auto k : {"k12","nope",""}) {
|
||||||
|
if (!probe(big_sel{}, k, big_sel::size())) { return false; }
|
||||||
|
}
|
||||||
|
TEST_SUCCEED();
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
bool run() {
|
bool run() {
|
||||||
return
|
return
|
||||||
object_find_field_unordered() &&
|
object_find_field_unordered() &&
|
||||||
@@ -181,6 +393,16 @@ namespace object_tests {
|
|||||||
object_find_field() &&
|
object_find_field() &&
|
||||||
document_object_find_field() &&
|
document_object_find_field() &&
|
||||||
value_object_find_field() &&
|
value_object_find_field() &&
|
||||||
|
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
object_find_field_key_selector() &&
|
||||||
|
object_for_each_callback_error() &&
|
||||||
|
key_selector_matchers_agree() &&
|
||||||
|
#endif
|
||||||
|
#if SIMDJSON_EXCEPTIONS && SIMDJSON_SUPPORTS_CONCEPTS
|
||||||
|
key_selector_example_toplevel() &&
|
||||||
|
key_selector_example_nested() &&
|
||||||
|
key_selector_example_inner_object() &&
|
||||||
|
#endif
|
||||||
true;
|
true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user