Minimal tag_invokes for STL types

This commit is contained in:
M. Bahoosh
2024-09-26 09:27:38 -10:00
parent 49b9860899
commit bbb3cb80a6
14 changed files with 347 additions and 128 deletions
@@ -3,8 +3,8 @@
#endif
// Stuff other things depend on
#include "simdjson/generic/ondemand/deserialize.h"
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/deserialize.h"
#include "simdjson/generic/ondemand/value_iterator.h"
#include "simdjson/generic/ondemand/value.h"
#include "simdjson/generic/ondemand/logger.h"
@@ -42,4 +42,5 @@
#include "simdjson/generic/ondemand/token_iterator-inl.h"
#include "simdjson/generic/ondemand/value_iterator-inl.h"
#include "simdjson/generic/ondemand/tag_invoke.h"
// Conversions
#include "simdjson/std/std.h"
+33 -13
View File
@@ -1,6 +1,7 @@
#ifndef SIMDJSON_ONDEMAND_DESERIALIZE_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_ONDEMAND_DESERIALIZE_H
#include "simdjson/generic/ondemand/base.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
#ifdef __has_include
#if __has_include(<version>)
@@ -15,6 +16,9 @@
namespace simdjson {
#ifdef __cpp_concepts
#define SIMDJSON_SUPPORTS_DESERIALIZATION 1
namespace tag_invoke_fn_ns {
void tag_invoke();
@@ -60,20 +64,36 @@ using tag_invoke_result_t =
template <auto &Tag> using tag_t = std::decay_t<decltype(Tag)>;
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
class value;
class document;
}
} // namespace SIMDJSON_IMPLEMENTATION
struct deserialize_tag;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept deserializable = tag_invocable<deserialize_tag, ValT&, T&>;
/// These types are deserializable in a built-in way
template <typename> struct is_builtin_deserializable : std::false_type {};
template <> struct is_builtin_deserializable<int64_t> : std::true_type {};
template <> struct is_builtin_deserializable<uint64_t> : std::true_type {};
template <> struct is_builtin_deserializable<double> : std::true_type {};
template <> struct is_builtin_deserializable<bool> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::array> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::object> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::value> : std::true_type {};
template <> struct is_builtin_deserializable<SIMDJSON_IMPLEMENTATION::ondemand::raw_json_string> : std::true_type {};
template <> struct is_builtin_deserializable<std::string_view> : std::true_type {};
template <typename T>
concept is_builtin_deserializable_v = is_builtin_deserializable<T>::value;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept nothrow_deserializable = nothrow_tag_invocable<deserialize_tag, ValT&, T&>;
concept custom_deserializable = tag_invocable<deserialize_tag, ValT&, T&>;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept deserializable = custom_deserializable<T, ValT> || is_builtin_deserializable_v<T>;
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept nothrow_custom_deserializable = nothrow_tag_invocable<deserialize_tag, ValT&, T&>;
// built-in types are noexcept and if an error happens, the value simply gets ignored and the error is returned.
template <typename T, typename ValT = SIMDJSON_IMPLEMENTATION::ondemand::value>
concept nothrow_deserializable = nothrow_custom_deserializable<T, ValT> || is_builtin_deserializable_v<T>;
/// Deserialize Tag
inline constexpr struct deserialize_tag {
@@ -82,15 +102,15 @@ inline constexpr struct deserialize_tag {
// Customization Point for value
template <typename T>
requires deserializable<T, value_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_deserializable<T, value_type>) {
requires custom_deserializable<T, value_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(value_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, value_type>) {
return tag_invoke(*this, object, output);
}
// Customization Point for document
template <typename T>
requires deserializable<T, document_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_deserializable<T, document_type>) {
requires custom_deserializable<T, document_type>
[[nodiscard]] constexpr /* error_code */ auto operator()(document_type &object, T& output) const noexcept(nothrow_custom_deserializable<T, document_type>) {
return tag_invoke(*this, object, output);
}
+4 -4
View File
@@ -184,7 +184,7 @@ public:
template <typename T>
simdjson_inline simdjson_result<T> get() &
#ifdef __cpp_concepts
noexcept(deserializable<T, document> ? nothrow_deserializable<T, document> : true)
noexcept(custom_deserializable<T, document> ? nothrow_custom_deserializable<T, document> : true)
#else
noexcept
#endif
@@ -207,7 +207,7 @@ public:
template<typename T>
simdjson_inline simdjson_result<T> get() &&
#ifdef __cpp_concepts
noexcept(deserializable<T, document> ? nothrow_deserializable<T, document> : true)
noexcept(custom_deserializable<T, document> ? nothrow_custom_deserializable<T, document> : true)
#else
noexcept
#endif
@@ -230,13 +230,13 @@ public:
template<typename T>
simdjson_inline error_code get(T &out) &
#ifdef __cpp_concepts
noexcept(deserializable<T, document> ? nothrow_deserializable<T, document> : true)
noexcept(custom_deserializable<T, document> ? nothrow_custom_deserializable<T, document> : true)
#else
noexcept
#endif
{
#ifdef __cpp_concepts
if constexpr (deserializable<T, document>) {
if constexpr (custom_deserializable<T, document>) {
return deserialize(*this, out);
} else {
#endif // __cpp_concepts
@@ -1,96 +0,0 @@
#ifndef SIMDJSON_TAG_INVOKE_H
#ifndef SIMDJSON_CONDITIONAL_INCLUDE
#define SIMDJSON_TAG_INVOKE_H
#include "simdjson/generic/ondemand/base.h"
#include "simdjson/generic/ondemand/deserialize.h"
#endif // SIMDJSON_CONDITIONAL_INCLUDE
#ifdef __has_include
#if __has_include(<version>)
#include <version>
#endif
#endif
#ifdef __cpp_concepts
#include <utility>
#include <vector>
#include <list>
/**
* Provides convenience functions for deserialization of common types.
*/
namespace simdjson {
namespace SIMDJSON_IMPLEMENTATION {
namespace ondemand {
template <std::unsigned_integral T>
error_code tag_invoke(deserialize_tag, auto &val, T& out) noexcept {
uint64_t x;
SIMDJSON_TRY(val.get_uint64().get(x));
if(x > (std::numeric_limits<T>::max)() || x < (std::numeric_limits<T>::min)()) {
return NUMBER_OUT_OF_RANGE;
}
out = static_cast<T>(x);
return SUCCESS;
}
template <std::floating_point T>
error_code tag_invoke(deserialize_tag, auto &val, T& out) noexcept {
double x;
SIMDJSON_TRY(val.get_double().get(x));
out = static_cast<T>(x);
return SUCCESS;
}
template <std::signed_integral T>
error_code tag_invoke(deserialize_tag, auto &val, T& out) noexcept {
int64_t x;
SIMDJSON_TRY(val.get_int64().get(x));
if(x > (std::numeric_limits<T>::max)() || x < (std::numeric_limits<T>::min)()) {
return NUMBER_OUT_OF_RANGE;
}
out = static_cast<T>(x);
return SUCCESS;
}
error_code tag_invoke(deserialize_tag, auto &val, std::string& out) noexcept {
SIMDJSON_TRY(val.get_string(out));
return SUCCESS;
}
/**
* STL containers have several constructors including one that takes a single
* size argument. Thus some compilers (Visual Studio) will not be able to
* disambiguate between the size and container constructor. Users should
* explicitly specify the type of the container as needed: e.g.,
* doc.get<std::vector<int>>().
*/
template <typename T, typename AllocT = std::allocator<T>>
error_code tag_invoke(deserialize_tag, auto &val, std::vector<T, AllocT>& out) noexcept {
array array;
SIMDJSON_TRY(val.get_array().get(array));
for (auto v : array) {
T value;
SIMDJSON_TRY(v.get<T>().get(value));
out.push_back(value);
}
return SUCCESS;
}
template <typename T, typename AllocT = std::allocator<T>>
error_code tag_invoke(deserialize_tag, auto &val, std::list<T, AllocT>& out) noexcept {
array array;
SIMDJSON_TRY(val.get_array().get(array));
for (auto v : array) {
T value;
SIMDJSON_TRY(v.get<T>().get(value));
out.push_back(value);
}
return SUCCESS;
}
}
}
}
#endif // __cpp_concepts
#endif // SIMDJSON_TAG_INVOKE_H
+3 -3
View File
@@ -41,7 +41,7 @@ public:
template <typename T>
simdjson_inline simdjson_result<T> get()
#ifdef __cpp_concepts
noexcept(deserializable<T, value> ? nothrow_deserializable<T, value> : true)
noexcept(custom_deserializable<T, value> ? nothrow_custom_deserializable<T, value> : true)
#else
noexcept
#endif
@@ -65,13 +65,13 @@ public:
template <typename T>
simdjson_inline error_code get(T &out)
#ifdef __cpp_concepts
noexcept(deserializable<T, value> ? nothrow_deserializable<T, value> : true)
noexcept(custom_deserializable<T, value> ? nothrow_custom_deserializable<T, value> : true)
#else
noexcept
#endif
{
#ifdef __cpp_concepts
if constexpr (deserializable<T, value>) {
if constexpr (custom_deserializable<T, value>) {
return deserialize(*this, out);
} else {
#endif // __cpp_concepts
+45
View File
@@ -0,0 +1,45 @@
#ifndef SIMDJSON_STD_LIST_H
#define SIMDJSON_STD_LIST_H
#ifndef SIMDJSON_SUPPORTS_DESERIALIZATION
#include "simdjson/generic/ondemand/deserialize.h"
#endif
#ifdef SIMDJSON_SUPPORTS_DESERIALIZATION
#include <list>
namespace simdjson {
template <typename T, typename AllocT, typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val,
std::list<T, AllocT> &out) noexcept(false) {
// For better error messages, don't use these as constraints on
// the tag_invoke CPO.
static_assert(
deserializable<T, ValT>,
"The specified type inside the list must itself be deserializable");
static_assert(
std::is_default_constructible_v<T>,
"The specified type inside the list must default constructible.");
ondemand::array arr;
SIMDJSON_TRY(val.get_array().get(arr));
for (auto v : arr) {
if (auto const err = v.get<T>().get(out.emplace_back()); err) {
// If an error occurs, the empty element that we just inserted gets
// removed. We're not using a temp variable because if T is a heavy type,
// we want the valid path to be the fast path and the slow path be the
// path that has errors in it.
static_cast<void>(out.pop_back());
return err;
}
}
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
#endif // SIMDJSON_STD_LIST_H
+55
View File
@@ -0,0 +1,55 @@
#ifndef SIMDJSON_STD_MEMORY_H
#define SIMDJSON_STD_MEMORY_H
#ifndef SIMDJSON_SUPPORTS_DESERIALIZATION
#include "simdjson/generic/ondemand/deserialize.h"
#endif
#ifdef SIMDJSON_SUPPORTS_DESERIALIZATION
#include <memory>
namespace simdjson {
/**
* This CPO (Customization Point Object) will help deserialize into
* `unique_ptr`s.
*
* If constructing T is nothrow, this conversion should be nothrow as well since
* we return MEMALLOC if we're not able to allocate memory instead of throwing
* the the error message.
*
* @tparam T The type inside the unique_ptr
* @tparam Deleter The Deleter of the unique_ptr
* @tparam ValT document/value type
* @param val document/value
* @param out output unique_ptr
* @return status of the conversion
*/
template <typename T, typename Deleter, typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val,
std::unique_ptr<T, Deleter>
&out) noexcept(nothrow_deserializable<T, ValT>) {
// For better error messages, don't use these as constraints on
// the tag_invoke CPO.
static_assert(
deserializable<T, ValT>,
"The specified type inside the unique_ptr must itself be deserializable");
static_assert(
std::is_default_constructible_v<T>,
"The specified type inside the unique_ptr must default constructible.");
auto ptr = new (std::nothrow) T();
if (ptr == nullptr) {
return MEMALLOC;
}
SIMDJSON_TRY(val.template get<T>(*ptr));
out.reset(ptr);
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
#endif // SIMDJSON_STD_MEMORY_H
+52
View File
@@ -0,0 +1,52 @@
#ifndef SIMDJSON_STD_STD_H
#define SIMDJSON_STD_STD_H
#ifndef SIMDJSON_SUPPORTS_DESERIALIZATION
#include "simdjson/generic/ondemand/deserialize.h"
#endif
#ifdef SIMDJSON_SUPPORTS_DESERIALIZATION
#include <concepts>
#include <limits>
namespace simdjson {
template <std::unsigned_integral T>
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept {
using limits = std::numeric_limits<T>;
uint64_t x;
SIMDJSON_TRY(val.get_uint64().get(x));
if (x > limits::max()) {
return NUMBER_OUT_OF_RANGE;
}
out = static_cast<T>(x);
return SUCCESS;
}
template <std::floating_point T>
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept {
double x;
SIMDJSON_TRY(val.get_double().get(x));
out = static_cast<T>(x);
return SUCCESS;
}
template <std::signed_integral T>
error_code tag_invoke(deserialize_tag, auto &val, T &out) noexcept {
using limits = std::numeric_limits<T>;
int64_t x;
SIMDJSON_TRY(val.get_int64().get(x));
if (x > limits::max() || x < limits::min()) {
return NUMBER_OUT_OF_RANGE;
}
out = static_cast<T>(x);
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
#endif // SIMDJSON_STD_STD_H
+37
View File
@@ -0,0 +1,37 @@
#ifndef SIMDJSON_STD_LIST_H
#define SIMDJSON_STD_LIST_H
#ifndef SIMDJSON_SUPPORTS_DESERIALIZATION
#include "simdjson/generic/ondemand/deserialize.h"
#endif
#ifdef SIMDJSON_SUPPORTS_DESERIALIZATION
#include <string>
namespace simdjson {
template <typename CharT,
typename TraitsT,
typename AllocT,
typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val, std::basic_string<CharT, TraitsT, AllocT> &out) noexcept(false) {
using string_type = std::basic_string<CharT, TraitsT, AllocT>;
if constexpr (std::same_as<string_type, string_type>) {
SIMDJSON_TRY(val.get_string(out));
} else {
// todo: optimize performance
std::string tmp;
SIMDJSON_TRY(val.get_string(tmp));
for (auto const ch : tmp) {
out.push_back(ch);
}
}
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
#endif // SIMDJSON_STD_LIST_H
+52
View File
@@ -0,0 +1,52 @@
#ifndef SIMDJSON_STD_VECTOR_H
#define SIMDJSON_STD_VECTOR_H
#ifndef SIMDJSON_SUPPORTS_DESERIALIZATION
#include "simdjson/generic/ondemand/deserialize.h"
#endif
#ifdef SIMDJSON_SUPPORTS_DESERIALIZATION
#include <vector>
namespace simdjson {
/**
* STL containers have several constructors including one that takes a single
* size argument. Thus, some compilers (Visual Studio) will not be able to
* disambiguate between the size and container constructor. Users should
* explicitly specify the type of the container as needed: e.g.,
* doc.get<std::vector<int>>().
*/
template <typename T, typename AllocT, typename ValT>
error_code tag_invoke(deserialize_tag, ValT &val,
std::vector<T, AllocT> &out) noexcept(false) {
// For better error messages, don't use these as constraints on
// the tag_invoke CPO.
static_assert(
deserializable<T, ValT>,
"The specified type inside the vector must itself be deserializable");
static_assert(
std::is_default_constructible_v<T>,
"The specified type inside the vector must default constructible.");
ondemand::array arr;
SIMDJSON_TRY(val.get_array().get(arr));
for (auto v : arr) {
if (auto const err = v.get<T>().get(out.emplace_back()); err) {
// If an error occurs, the empty element that we just inserted gets
// removed. We're not using a temp variable because if T is a heavy type,
// we want the valid path to be the fast path and the slow path be the
// path that has errors in it.
static_cast<void>(out.pop_back());
return err;
}
}
return SUCCESS;
}
} // namespace simdjson
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
#endif // SIMDJSON_STD_VECTOR_H
+1
View File
@@ -30,6 +30,7 @@ add_cpp_test(ondemand_wrong_type_error_tests LABELS ondemand acceptance
add_cpp_test(ondemand_iterate_many_csv LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_custom_types_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_custom_types_document_tests LABELS ondemand acceptance per_implementation)
add_cpp_test(ondemand_stl_types_tests LABELS ondemand acceptance per_implementation)
if(NOT SIMDJSON_SANITIZE)
add_cpp_test(ondemand_cacheline LABELS ondemand acceptance per_implementation)
endif()
@@ -1,5 +1,6 @@
#include "simdjson.h"
#include "test_ondemand.h"
#include "simdjson/std/vector.h"
#include <string>
#include <vector>
@@ -63,10 +64,10 @@ struct Car {
}
};
static_assert(simdjson::deserializable<std::unique_ptr<Car>, simdjson::ondemand::value>, "It should be invocable");
static_assert(simdjson::deserializable<std::unique_ptr<Car>, simdjson::ondemand::document>, "Tag_invoke should work with document as well.");
static_assert(simdjson::deserializable<std::vector<Car>, simdjson::ondemand::value>, "It should be invocable");
static_assert(simdjson::deserializable<std::vector<Car>, simdjson::ondemand::document>, "Tag_invoke should work with document as well.");
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>, simdjson::ondemand::value>, "It should be invocable");
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>, simdjson::ondemand::document>, "Tag_invoke should work with document as well.");
static_assert(simdjson::custom_deserializable<std::vector<Car>, simdjson::ondemand::value>, "It should be invocable");
static_assert(simdjson::custom_deserializable<std::vector<Car>, simdjson::ondemand::document>, "Tag_invoke should work with document as well.");
bool custom_test() {
TEST_START();
@@ -1,10 +1,11 @@
#include "simdjson.h"
#include "simdjson/std/vector.h"
#include "test_ondemand.h"
#include <string>
#include <vector>
#ifdef __cpp_concepts
#ifdef SIMDJSON_SUPPORTS_DESERIALIZATION
template <typename T>
struct is_unique_ptr : std::false_type {
@@ -27,7 +28,7 @@ namespace simdjson {
// This tag_invoke MUST be inside simdjson namespace
template <typename T>
requires is_unique_ptr_v<T>
auto tag_invoke(deserialize_tag, ondemand::value &val, T& out) {
auto tag_invoke(deserialize_tag, auto &val, T& out) {
using type = typename T::element_type;
out = std::make_unique<type>(val.template get<type>());
return SUCCESS;
@@ -36,11 +37,11 @@ auto tag_invoke(deserialize_tag, ondemand::value &val, T& out) {
} // namespace simdjson
#endif // __cpp_concepts
#endif // SIMDJSON_SUPPORTS_DESERIALIZATION
namespace custom_types_tests {
#if SIMDJSON_EXCEPTIONS && defined(__cpp_concepts)
#if SIMDJSON_EXCEPTIONS && defined(SIMDJSON_SUPPORTS_DESERIALIZATION)
struct Car {
std::string make{};
std::string model{};
@@ -87,7 +88,7 @@ struct Car {
}
};
static_assert(simdjson::deserializable<std::unique_ptr<Car>>, "It should be deserializable");
static_assert(simdjson::custom_deserializable<std::unique_ptr<Car>>, "It should be deserializable");
bool custom_uniqueptr_test() {
TEST_START();
@@ -214,7 +215,7 @@ bool custom_no_except() {
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS && defined(__cpp_concepts)
#if SIMDJSON_EXCEPTIONS && defined(SIMDJSON_SUPPORTS_DESERIALIZATION)
custom_test() &&
custom_uniqueptr_test() &&
custom_no_except() &&
@@ -0,0 +1,50 @@
#include "simdjson.h"
#include "simdjson/std/memory.h"
#include "simdjson/std/vector.h"
#include "test_ondemand.h"
#include <string>
#include <vector>
namespace stl_types {
#if SIMDJSON_EXCEPTIONS && defined(SIMDJSON_SUPPORTS_DESERIALIZATION)
bool basic_general_madness() {
TEST_START();
simdjson::padded_string json =
R"( [ { "codes": [1.2, 3.4, 5.6, 7.8, 9.0] },
{ "codes": [2.2, 4.4, 6.6, 8.8, 10.0] } ])"_padded;
simdjson::ondemand::parser parser;
simdjson::ondemand::document doc = parser.iterate(json);
std::vector<std::unique_ptr<float>> codes;
for (auto val : doc) {
simdjson::ondemand::object obj;
SIMDJSON_TRY(val.get_object().get(obj));
obj["codes"].get(codes); // append to it
}
if (codes.size() != 10) {
return false;
}
if (*codes[5] != 2.2f || *codes[7] != 6.6f || *codes[9] != 10.0f) {
return false;
}
TEST_SUCCEED();
}
#endif // SIMDJSON_EXCEPTIONS
bool run() {
return
#if SIMDJSON_EXCEPTIONS && defined(SIMDJSON_SUPPORTS_DESERIALIZATION)
basic_general_madness() &&
#endif // SIMDJSON_EXCEPTIONS
true;
}
} // namespace stl_types
int main(int argc, char *argv[]) {
return test_main(argc, argv, stl_types::run);
}