mirror of
https://github.com/simdjson/simdjson
synced 2026-06-08 17:27:07 +00:00
init
This commit is contained in:
@@ -59,6 +59,14 @@ concept appendable_containers =
|
||||
details::supports_add<T> || details::supports_append<T> ||
|
||||
details::supports_insert<T>) && !string_view_keyed_map<T>;
|
||||
|
||||
/// Check if T is a key_selector type for efficient JSON field lookup
|
||||
template <typename T>
|
||||
concept key_selector_type = requires(T selector) {
|
||||
{ selector.size() } -> std::same_as<std::size_t>;
|
||||
{ selector.index_of(std::string_view{}) } -> std::same_as<std::size_t>;
|
||||
{ selector.get_key(std::size_t{}) } -> std::same_as<std::string_view>;
|
||||
};
|
||||
|
||||
/// Insert into the container however possible
|
||||
template <appendable_containers T, typename... Args>
|
||||
constexpr decltype(auto) emplace_one(T &vec, Args &&...args) {
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#include "simdjson/generic/ondemand/document.h"
|
||||
#include "simdjson/generic/ondemand/document_stream.h"
|
||||
#include "simdjson/generic/ondemand/field.h"
|
||||
#include "simdjson/generic/ondemand/key_selector.h"
|
||||
#include "simdjson/generic/ondemand/object.h"
|
||||
#include "simdjson/generic/ondemand/object_iterator.h"
|
||||
#include "simdjson/generic/ondemand/ranges.h"
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
#ifndef SIMDJSON_GENERIC_ONDEMAND_KEY_SELECTOR_H
|
||||
#define SIMDJSON_GENERIC_ONDEMAND_KEY_SELECTOR_H
|
||||
|
||||
#include "simdjson/base.h"
|
||||
#include <array>
|
||||
#include <string_view>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
|
||||
namespace simdjson {
|
||||
namespace SIMDJSON_IMPLEMENTATION {
|
||||
namespace ondemand {
|
||||
|
||||
|
||||
|
||||
// Forward declaration
|
||||
class object;
|
||||
|
||||
/**
|
||||
* A compile-time key selector for efficient JSON object field lookup.
|
||||
* Uses perfect hashing (gperf-style) to map keys to identifiers.
|
||||
*/
|
||||
template <std::size_t N>
|
||||
class key_selector {
|
||||
static_assert(N > 0, "key_selector requires at least one key");
|
||||
static_assert(N <= 100, "key_selector supports at most 100 keys");
|
||||
|
||||
// Perfect hash table data
|
||||
std::array<std::array<std::uint8_t, 256>, 16> asso_values_{};
|
||||
std::uint8_t num_positions_{};
|
||||
std::array<std::uint8_t, 16> positions_{};
|
||||
std::array<std::uint8_t, 100> slot_to_key_{}; // Max table size
|
||||
std::array<std::uint8_t, N> key_to_slot_{};
|
||||
std::array<std::array<char, 64>, N> key_data_{};
|
||||
std::array<std::uint8_t, N> key_lengths_{};
|
||||
std::size_t table_size_{};
|
||||
|
||||
public:
|
||||
// Validate keys at compile time
|
||||
constexpr void validate_keys(const std::array<std::string_view, N>& keys) {
|
||||
for (std::size_t i = 0; i < N; ++i) {
|
||||
auto key = keys[i];
|
||||
if (key.empty()) {
|
||||
throw "Empty keys are not allowed in key_selector";
|
||||
}
|
||||
if (key.size() > SIMDJSON_PADDING) {
|
||||
throw "Key length exceeds SIMDJSON_PADDING (64 bytes)";
|
||||
}
|
||||
for (char c : key) {
|
||||
if (c == '\\') {
|
||||
throw "Escape characters (\\) are not allowed in key_selector keys";
|
||||
}
|
||||
if (c == '\0') {
|
||||
throw "Null characters are not allowed in key_selector keys";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Simple perfect hash generation using polynomial rolling hash
|
||||
constexpr void generate_hash_table(const std::array<std::string_view, N>& keys) {
|
||||
// Use polynomial hash: hash = sum(key[i] * 31^(len-1-i)) % table_size
|
||||
const std::size_t prime = 31;
|
||||
|
||||
// Find minimum table size that works
|
||||
std::size_t table_size = N;
|
||||
bool success = false;
|
||||
while (!success && table_size <= 100) { // Max table size
|
||||
success = true;
|
||||
std::array<std::uint8_t, 100> used_slots{};
|
||||
std::fill(used_slots.begin(), used_slots.begin() + table_size, 0);
|
||||
// Initialize slot_to_key_ to invalid values
|
||||
std::fill(slot_to_key_.begin(), slot_to_key_.begin() + table_size, static_cast<std::uint8_t>(N));
|
||||
|
||||
for (std::size_t i = 0; i < N && success; ++i) {
|
||||
std::size_t hash = 0;
|
||||
std::size_t power = 1;
|
||||
for (std::size_t j = keys[i].size(); j > 0; --j) {
|
||||
unsigned char ch = static_cast<unsigned char>(keys[i][j-1]);
|
||||
hash = (hash + ch * power) % table_size;
|
||||
power = (power * prime) % table_size;
|
||||
}
|
||||
std::size_t slot = hash;
|
||||
|
||||
if (used_slots[slot]) {
|
||||
success = false;
|
||||
} else {
|
||||
used_slots[slot] = 1;
|
||||
slot_to_key_[slot] = static_cast<std::uint8_t>(i);
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
table_size++;
|
||||
}
|
||||
}
|
||||
|
||||
table_size_ = table_size;
|
||||
|
||||
// Build key_to_slot mapping
|
||||
for (std::size_t slot = 0; slot < table_size_; ++slot) {
|
||||
std::uint8_t key_idx = slot_to_key_[slot];
|
||||
if (key_idx < N) {
|
||||
key_to_slot_[key_idx] = static_cast<std::uint8_t>(slot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
constexpr key_selector(const std::array<std::string_view, N>& keys) {
|
||||
validate_keys(keys);
|
||||
|
||||
// Store key data
|
||||
for (std::size_t i = 0; i < N; ++i) {
|
||||
key_lengths_[i] = static_cast<std::uint8_t>(keys[i].size());
|
||||
std::copy(keys[i].begin(), keys[i].end(), key_data_[i].begin());
|
||||
}
|
||||
|
||||
generate_hash_table(keys);
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr std::size_t size() const noexcept { return N; }
|
||||
|
||||
[[nodiscard]] constexpr std::size_t compute_hash(std::string_view key) const noexcept {
|
||||
const std::size_t prime = 31;
|
||||
std::size_t hash = 0;
|
||||
std::size_t power = 1;
|
||||
for (std::size_t j = key.size(); j > 0; --j) {
|
||||
unsigned char ch = static_cast<unsigned char>(key[j-1]);
|
||||
hash = (hash + ch * power) % table_size_;
|
||||
power = (power * prime) % table_size_;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr bool contains(std::string_view key) const noexcept {
|
||||
std::size_t slot = compute_hash(key);
|
||||
if (slot >= table_size_) return false;
|
||||
|
||||
std::uint8_t key_idx = slot_to_key_[slot];
|
||||
if (key_idx >= N) return false;
|
||||
|
||||
// Compare key
|
||||
if (key_lengths_[key_idx] != key.size()) return false;
|
||||
return std::equal(key.begin(), key.end(), key_data_[key_idx].begin());
|
||||
}
|
||||
|
||||
[[nodiscard]] constexpr std::size_t index_of(std::string_view key) const noexcept {
|
||||
std::size_t slot = compute_hash(key);
|
||||
if (slot >= table_size_) return N; // Invalid index
|
||||
|
||||
std::uint8_t key_idx = slot_to_key_[slot];
|
||||
if (key_idx >= N) return N;
|
||||
|
||||
// Compare key
|
||||
if (key_lengths_[key_idx] != key.size()) return N;
|
||||
if (!std::equal(key.begin(), key.end(), key_data_[key_idx].begin())) return N;
|
||||
|
||||
return key_idx;
|
||||
}
|
||||
|
||||
// Accessors for key data (used by object::find_field)
|
||||
[[nodiscard]] constexpr std::string_view get_key(std::size_t index) const noexcept {
|
||||
if (index >= N) return {};
|
||||
return std::string_view(key_data_[index].data(), key_lengths_[index]);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace ondemand
|
||||
} // namespace SIMDJSON_IMPLEMENTATION
|
||||
} // namespace simdjson
|
||||
|
||||
#endif // SIMDJSON_SUPPORTS_CONCEPTS
|
||||
|
||||
#endif // SIMDJSON_GENERIC_ONDEMAND_KEY_SELECTOR_H
|
||||
@@ -63,6 +63,42 @@ simdjson_inline simdjson_result<value> object::find_field(const std::string_view
|
||||
return value(iter.child());
|
||||
}
|
||||
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<value>> object::find_field(const Selector& selector) & noexcept {
|
||||
// Try to find any of the keys in the selector
|
||||
for (std::size_t i = 0; i < selector.size(); ++i) {
|
||||
std::string_view key = selector.get_key(i);
|
||||
auto result = iter.find_field_unordered_raw(key);
|
||||
if (result.error()) {
|
||||
return {selector.size(), result.error()};
|
||||
}
|
||||
bool has_value = result.value();
|
||||
if (has_value) {
|
||||
return {i, value(iter.child())};
|
||||
}
|
||||
}
|
||||
return {selector.size(), NO_SUCH_FIELD}; // Return size() as invalid index
|
||||
}
|
||||
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<value>> object::find_field(const Selector& selector) && noexcept {
|
||||
// Try to find any of the keys in the selector
|
||||
for (std::size_t i = 0; i < selector.size(); ++i) {
|
||||
std::string_view key = selector.get_key(i);
|
||||
auto result = iter.find_field_unordered_raw(key);
|
||||
if (result.error()) {
|
||||
return {selector.size(), result.error()};
|
||||
}
|
||||
bool has_value = result.value();
|
||||
if (has_value) {
|
||||
return {i, value(iter.child())};
|
||||
}
|
||||
}
|
||||
return {selector.size(), NO_SUCH_FIELD}; // Return size() as invalid index
|
||||
}
|
||||
#endif
|
||||
|
||||
simdjson_inline simdjson_result<object> object::start(value_iterator &iter) noexcept {
|
||||
SIMDJSON_TRY( iter.start_object().error() );
|
||||
return object(iter);
|
||||
@@ -326,6 +362,20 @@ simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjs
|
||||
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first).find_field(key);
|
||||
}
|
||||
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::find_field(const Selector& selector) & noexcept {
|
||||
if (error()) { return {0, error()}; }
|
||||
return first.find_field(selector);
|
||||
}
|
||||
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::find_field(const Selector& selector) && noexcept {
|
||||
if (error()) { return {0, error()}; }
|
||||
return std::forward<SIMDJSON_IMPLEMENTATION::ondemand::object>(first).find_field(selector);
|
||||
}
|
||||
#endif
|
||||
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::object>::at_pointer(std::string_view json_pointer) noexcept {
|
||||
if (error()) { return error(); }
|
||||
return first.at_pointer(json_pointer);
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "simdjson/generic/ondemand/base.h"
|
||||
#include "simdjson/generic/implementation_simdjson_result_base.h"
|
||||
#include "simdjson/generic/ondemand/value_iterator.h"
|
||||
#include "simdjson/generic/ondemand/key_selector.h"
|
||||
#include <vector>
|
||||
#if SIMDJSON_STATIC_REFLECTION && SIMDJSON_SUPPORTS_CONCEPTS
|
||||
#include "simdjson/generic/ondemand/json_string_builder.h" // for constevalutil::fixed_string
|
||||
@@ -122,6 +123,23 @@ public:
|
||||
/** @overload simdjson_inline simdjson_result<value> find_field_unordered(std::string_view key) & noexcept; */
|
||||
simdjson_inline simdjson_result<value> operator[](std::string_view key) && noexcept;
|
||||
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
/**
|
||||
* Look up a field by name using a key_selector. This method is similar to find_field_unordered()
|
||||
* but uses a compile-time generated perfect hash table for efficient lookup.
|
||||
*
|
||||
* @tparam Selector The key_selector type
|
||||
* @param selector The key selector instance
|
||||
* @returns A pair containing the key identifier (index in the selector) and the value,
|
||||
* or NO_SUCH_FIELD if the field is not in the object.
|
||||
*/
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<value>> find_field(const Selector& selector) & noexcept;
|
||||
/** @overload template <concepts::key_selector_type Selector> simdjson_inline std::pair<std::size_t, simdjson_result<value>> find_field(const Selector& selector) & noexcept; */
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<value>> find_field(const Selector& selector) && noexcept;
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Get the value associated with the given JSON pointer. We use the RFC 6901
|
||||
* https://tools.ietf.org/html/rfc6901 standard, interpreting the current node
|
||||
@@ -332,6 +350,12 @@ public:
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field(std::string_view key) && noexcept;
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) & noexcept;
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> find_field_unordered(std::string_view key) && noexcept;
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>> find_field(const Selector& selector) & noexcept;
|
||||
template <concepts::key_selector_type Selector>
|
||||
simdjson_inline std::pair<std::size_t, simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value>> find_field(const Selector& selector) && noexcept;
|
||||
#endif
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) & noexcept;
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> operator[](std::string_view key) && noexcept;
|
||||
simdjson_inline simdjson_result<SIMDJSON_IMPLEMENTATION::ondemand::value> at_pointer(std::string_view json_pointer) noexcept;
|
||||
|
||||
@@ -26,6 +26,7 @@ add_cpp_test(ondemand_nan_inf_tests LABELS ondemand acceptance
|
||||
add_cpp_test(ondemand_number_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_number_in_string_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_object_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_object_find_field_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_object_error_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_ordering_tests LABELS ondemand acceptance per_implementation)
|
||||
add_cpp_test(ondemand_parse_api_tests LABELS ondemand acceptance per_implementation)
|
||||
|
||||
@@ -173,6 +173,36 @@ namespace object_tests {
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
bool object_find_field_key_selector() {
|
||||
TEST_START();
|
||||
auto json = R"({ "name": "John", "age": 30, "city": "New York" })"_padded;
|
||||
constexpr std::array<std::string_view, 3> keys = {"name", "age", "city"};
|
||||
constexpr auto selector = ondemand::key_selector<3>(keys);
|
||||
|
||||
SUBTEST("ondemand::object with key_selector", test_ondemand_doc(json, [&](auto doc_result) {
|
||||
ondemand::object object;
|
||||
ASSERT_SUCCESS( doc_result.get(object) );
|
||||
|
||||
auto [index, value_result] = object.find_field(selector);
|
||||
ASSERT_TRUE(index < 3);
|
||||
ASSERT_SUCCESS(value_result);
|
||||
std::string_view str_val;
|
||||
ASSERT_SUCCESS(value_result.get(str_val));
|
||||
ASSERT_EQUAL(str_val, "John");
|
||||
|
||||
// Test that we can find different keys
|
||||
ASSERT_EQUAL(selector.index_of("name"), 0);
|
||||
ASSERT_EQUAL(selector.index_of("age"), 1);
|
||||
ASSERT_EQUAL(selector.index_of("city"), 2);
|
||||
ASSERT_EQUAL(selector.index_of("invalid"), 3); // Not found
|
||||
|
||||
return true;
|
||||
}));
|
||||
TEST_SUCCEED();
|
||||
}
|
||||
#endif
|
||||
|
||||
bool run() {
|
||||
return
|
||||
object_find_field_unordered() &&
|
||||
@@ -181,6 +211,9 @@ namespace object_tests {
|
||||
object_find_field() &&
|
||||
document_object_find_field() &&
|
||||
value_object_find_field() &&
|
||||
#if SIMDJSON_SUPPORTS_CONCEPTS
|
||||
object_find_field_key_selector() &&
|
||||
#endif
|
||||
true;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user